chore(IDX): Simplify versioning (#3984)

This simplifies the mechanism used to stamp the released artifacts with
a version.

In the non-release mode, a constant version is used. In release mode
(with `--config=stamp`) the HEAD commit id and timestamp are used.

This also removes some workarounds that were necessary to re-trigger
bazel evaluation (timestamp file, etc) as well as the distinction
between "IC version" and "IC RC-only version".

Moreover the upload check (to ensure we don't upload non-release
artifacts) is removed because (1) uploads are (currently) always
necessary (for SHA256SUMS files) and (2) the check was using an
incorrect string in the comparison (couple extra zeros) which meant it
never triggers.

.gitignore

@@ -113,7 +113,6 @@ artifacts
 bazel-build-log.json
 bazel-bep.pb
 bazel-bep.txt
-bazel-timestamp.txt
 user.bazelrc
 
 # Bazel profile

BUILD.bazel

@@ -29,15 +29,6 @@ filegroup(
     srcs = ["mainnet-subnet-revisions.json"],
 )
 
-# bazel/workspace_status.sh will write the current timestamp to this file on every run to provade an input that always changes for some targets.
-# The file does not have to and should not be committed, therefore will not exist on the first run on a clean source tree.
-# Therefore glob is used to not fail when the file does not exist.
-# buildifier: disable=constant-glob
-filegroup(
-    name = "bazel-timestamp",
-    srcs = glob(["bazel-timestamp.txt"]),
-)
-
 alias(
     name = "buildifier",
     actual = "//bazel:buildifier",

bazel/BUILD.bazel

@@ -1,6 +1,6 @@
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag")
 load("@buildifier_prebuilt//:rules.bzl", "buildifier")
-load("//bazel:status.bzl", "FAKE_IC_VERSION", "ic_version_or_git_sha")
+load("//bazel:defs.bzl", "write_info_file_var")
 
 bool_flag(
     name = "enable_malicious_code",
@@ -40,62 +40,18 @@ config_setting(
     },
 )
 
-string_flag(
-    name = "ic_version",
-    build_setting_default = "",
-)
-
-string_flag(
-    name = "ic_version_rc_only",
-    build_setting_default = FAKE_IC_VERSION,
-)
-
-bool_flag(
-    name = "release_build",
-    build_setting_default = False,
-)
-
-config_setting(
-    name = "is_release_build",
-    flag_values = {
-        ":release_build": "True",
-    },
-)
-
 string_flag(
     name = "timeout_value",
     build_setting_default = "10m",
     visibility = ["//visibility:public"],
 )
 
-# Generates version.txt file that contains the value of `--ic_version` flag if set, otherwise <git sha>-<timestamp>.
-# It has to be tagged as `no-cache` as bazel treats version_file as never changing.
-# Visibility of ic_version is limited to package group uploaders and subpackages.
-# Other packages should consume ic_version of artifacts they target.
-ic_version_or_git_sha(
+write_info_file_var(
     name = "version.txt",
-    ic_version = ":ic_version",
-    tags = ["no-cache"],
-    visibility = [
-        "__subpackages__",
-        "//ci/src/artifacts:uploaders",
-    ],
-)
-
-# Similar to the above target, but use the `ic_version_rc_only` flag. The intention is to set this flag to a dummy value on MR pipelines to preserve caching
-# but on master and release candidate branches set to the current git commit sha.
-ic_version_or_git_sha(
-    name = "rc_only_version.txt",
-    ic_version = ":ic_version_rc_only",
-    tags = ["no-cache"],
+    varname = "STABLE_VERSION",
     visibility = ["//visibility:public"],
 )
 
-sh_binary(
-    name = "ic_version_or_git_sha_sh",
-    srcs = ["ic_version_or_git_sha.sh"],
-)
-
 exports_files(
     [
         "prost_generator.sh",

bazel/canisters.bzl

@@ -126,7 +126,7 @@ def rust_canister(name, service_file, visibility = ["//visibility:public"], test
         name = final_name,
         src_wasm = wasm_name,
         service_file = service_file,
-        version_file = "//bazel:rc_only_version.txt",
+        version_file = "//bazel:version.txt",
         visibility = visibility,
         testonly = testonly,
         keep_name_section = keep_name_section,
@@ -177,7 +177,7 @@ def motoko_canister(name, entry, deps):
     finalize_wasm(
         name = name + ".wasm",
         src_wasm = raw_wasm,
-        version_file = "//bazel:rc_only_version.txt",
+        version_file = "//bazel:version.txt",
         testonly = False,
     )
 

bazel/conf/.bazelrc.build

@@ -39,18 +39,18 @@ build --strip=never
 # Build everything ic-os without sandbox
 build --strategy_regexp=ic-os[:/].*=local
 
-build --workspace_status_command=$(pwd)/bazel/workspace_status.sh
+build --workspace_status_command='$(pwd)/bazel/workspace_status.sh'
 
 build --experimental_repository_downloader_retries=3 # https://bazel.build/reference/command-line-reference#flag--experimental_repository_downloader_retries
 
-common --flag_alias=ic_version=//bazel:ic_version
-common --flag_alias=ic_version_rc_only=//bazel:ic_version_rc_only
 common --flag_alias=release_build=//bazel:release_build
 common --flag_alias=s3_endpoint=//ci/src/artifacts:s3_endpoint
 common --flag_alias=s3_upload=//ci/src/artifacts:s3_upload
 common --flag_alias=k8s=//rs/tests:k8s
 common --flag_alias=timeout_value=//bazel:timeout_value
 
+common:stamped --workspace_status_command='$(pwd)/bazel/workspace_status.sh --stamp'
+
 # Exclude system tests by default
 # https://github.com/bazelbuild/bazel/issues/8439
 build --build_tag_filters="-system_test,-upload,-fuzz_test"
@@ -68,7 +68,7 @@ test:precommit --build_tests_only --test_tag_filters="smoke"
 build:systest --build_tag_filters=
 test:systest --test_output=streamed --test_tag_filters=
 
-build:testnet --build_tag_filters= --ic_version_rc_only=
+build:testnet --build_tag_filters=
 test:testnet --test_output=streamed --test_tag_filters=
 
 # For sandboxed actions, mount an empty, writable directory at this absolute path

bazel/defs.bzl

@@ -380,3 +380,23 @@ symlink_dirs = rule(
         "targets": attr.label_keyed_string_dict(allow_files = True),
     },
 )
+
+def _write_info_file_var_impl(ctx):
+    """Helper rule that creates a file with the content of the provided var from the info file."""
+
+    output = ctx.actions.declare_file(ctx.label.name)
+    ctx.actions.run_shell(
+        command = """
+            grep <{info_file} -e '{varname}' \\
+                    | cut -d' ' -f2 > {out}""".format(varname = ctx.attr.varname, info_file = ctx.info_file.path, out = output.path),
+        inputs = [ctx.info_file],
+        outputs = [output],
+    )
+    return [DefaultInfo(files = depset([output]))]
+
+write_info_file_var = rule(
+    implementation = _write_info_file_var_impl,
+    attrs = {
+        "varname": attr.string(mandatory = True),
+    },
+)

bazel/workspace_status.sh

@@ -2,13 +2,22 @@
 
 set -euo pipefail
 
-# Used by ic_version_or_git_sha
-commit_sha=$(git rev-parse HEAD)
-echo "COMMIT_SHA $commit_sha"
-
-# Used by ic_version_or_git_sha
-git_tree_status=$(git diff-index --quiet HEAD -- && echo 'Clean' || echo 'Modified')
-echo "GIT_TREE_STATUS $git_tree_status"
+# By default, we set a hardcoded, constant version to avoid rebuilds. Only when
+# --stamp is provided do we write a meaningful version.
+if [ "$#" == "0" ]; then
+    echo "STABLE_VERSION 0000000000000000000000000000000000000000"
+    echo "STABLE_COMMIT_TIMESTAMP 4000000000" # arbitrary (constant) timestamp
+elif [ "$#" == "1" ] && [ "$1" == "--stamp" ]; then
+    version="$(git rev-parse HEAD)"
+    # If the checkout is not clean, mark the version as dirty
+    if [ -n "$(git status --porcelain)" ]; then
+        version="$version-dirty"
+    fi
+    echo "STABLE_VERSION $version"
+    echo "STABLE_COMMIT_TIMESTAMP $(git show -s --format=%ct)"
+else
+    exit 1
+fi
 
 # Used to read credentials for S3 upload
 echo "HOME ${HOME}"
@@ -20,7 +29,3 @@ if [[ -n "${USER:-}" ]]; then
 elif [[ -n "${HOSTUSER:-}" ]]; then
     echo "STABLE_FARM_USER ${HOSTUSER}"
 fi
-
-# Generate a file that changes every time bazel runs. It can be used as dependency for targets we want to always rebuild.
-workspace_root="$(git rev-parse --show-toplevel)"
-date '+%s' >"$workspace_root/bazel-timestamp.txt"

ci/bazel-scripts/main.sh

@@ -7,24 +7,22 @@
 set -eufo pipefail
 
 # default behavior is to build targets specified in BAZEL_TARGETS and not upload to s3
-ic_version_rc_only="0000000000000000000000000000000000000000"
 release_build="false"
 s3_upload="False"
 
+# List of "protected" branches, i.e. branches (not necessarily "protected" in the GitHub sense) where we need
+# the full build to occur (including versioning
 protected_branches=("^master$" "^rc--" "^hotfix-" "^master-private$")
-
-# if we are on a protected branch or targeting a rc branch we set ic_version to the commit_sha and upload to s3
 for pattern in "${protected_branches[@]}"; do
     if [[ "$BRANCH_NAME" =~ $pattern ]]; then
         IS_PROTECTED_BRANCH="true"
         break
     fi
 done
 
-# if we are on a protected branch or targeting a rc branch we set release build, ic_version to the commit_sha and
-# upload to s3
+# if we are on a "protected" branch or targeting a rc branch we upload all artifacts and run a release build
+# (with versioning)
 if [[ "${IS_PROTECTED_BRANCH:-}" == "true" ]]; then
-    ic_version_rc_only="${CI_COMMIT_SHA}"
     s3_upload="True"
     release_build="true"
     RUN_ON_DIFF_ONLY="false"
@@ -90,12 +88,13 @@ bazel_args=(
     ${BAZEL_TARGETS}
     --color=yes
     --build_metadata=BUILDBUDDY_LINKS="[CI Job](${CI_JOB_URL})"
-    --ic_version="${CI_COMMIT_SHA}"
-    --ic_version_rc_only="${ic_version_rc_only}"
-    --release_build="${release_build}"
     --s3_upload="${s3_upload:-"False"}"
 )
 
+if [[ $release_build == true ]]; then
+    bazel_args+=(--config=stamped)
+fi
+
 # Unless explicitly provided, we set a default --repository_cache to a volume mounted inside our runners
 # Only for Linux builds since there `/cache` is mounted to host local storage.
 if [[ ! " ${bazel_args[*]} " =~ [[:space:]]--repository_cache[[:space:]] ]] && [[ "$(uname)" == "Linux" ]]; then

ci/container/build-ic.sh

@@ -67,7 +67,7 @@ function join_by {
 export BUILD_BIN=false
 export BUILD_CAN=false
 export BUILD_IMG=false
-export RELEASE=true
+release_build=true
 
 if [ "$#" == 0 ]; then
     echo_red "ERROR: Please specify one of '-b', '-c' or '-i'" >&2
@@ -86,7 +86,7 @@ while getopts ':bcinh-:' OPT; do
         b | binaries) BUILD_BIN=true ;;
         c | canisters) BUILD_CAN=true ;;
         i | icos) BUILD_IMG=true ;;
-        n | non-release | no-release | norelease) RELEASE=false ;;
+        n | non-release | no-release | norelease) release_build=false ;;
         ??*) echo_red "Invalid option --$OPT" && usage && exit 1 ;;
         ?) echo_red "Invalid command option." && usage && exit 1 ;;
     esac
@@ -107,11 +107,16 @@ fi
 
 export VERSION="$(git rev-parse HEAD)"
 
-if "$RELEASE"; then
-    export IC_VERSION_RC_ONLY="$VERSION"
+BAZEL_TARGETS=()
+
+BAZEL_COMMON_ARGS=(
+    --config=local
+)
+
+if [[ $release_build == true ]]; then
     echo_red "Building release revision (master or rc--*)! Use '--no-release' for non-release revision!" && sleep 2
+    BAZEL_COMMON_ARGS+=(--config=stamped)
 else
-    export IC_VERSION_RC_ONLY="0000000000000000000000000000000000000000"
     echo_red "Building non-release revision!" && sleep 2
 fi
 
@@ -127,15 +132,6 @@ rm -rf "$BINARIES_DIR_FULL"
 rm -rf "$CANISTERS_DIR_FULL"
 rm -rf "$DISK_DIR_FULL"
 
-BAZEL_TARGETS=()
-
-BAZEL_COMMON_ARGS=(
-    --config=local
-    --ic_version="$VERSION"
-    --ic_version_rc_only="$IC_VERSION_RC_ONLY"
-    --release_build="$RELEASE"
-)
-
 if "$BUILD_BIN"; then BAZEL_TARGETS+=("//publish/binaries:compute_checksums"); fi
 if "$BUILD_CAN"; then BAZEL_TARGETS+=("//publish/canisters:compute_checksums"); fi
 if "$BUILD_IMG"; then BAZEL_TARGETS+=(

ci/src/artifacts/upload.bzl

@@ -3,7 +3,6 @@ Rules to manipulate with artifacts: download, upload etc.
 """
 
 load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
-load("//bazel:status.bzl", "FAKE_IC_VERSION")
 
 def _upload_artifact_impl(ctx):
     """
@@ -50,7 +49,6 @@ def _upload_artifact_impl(ctx):
                 "REMOTE_SUBDIR": ctx.attr.remote_subdir,
                 "VERSION_FILE": ctx.version_file.path,
                 "VERSION_TXT": ctx.file._version_txt.path,
-                "FAKE_IC_VERSION": FAKE_IC_VERSION,
             },
             inputs = [f, ctx.version_file, rclone_config, ctx.file._version_txt],
             outputs = [url],