Merge branch 'master' into alexuta/large_sector_size

.github/CODEOWNERS

@@ -44,16 +44,19 @@ go_deps.bzl               @dfinity/idx
 /publish/ @dfinity/idx
 
 # [Packages]
-/packages/icrc-cbor/                 @dfinity/finint @dfinity/cross-chain-team
-/packages/icrc-ledger-agent/         @dfinity/finint
-/packages/icrc-ledger-types/         @dfinity/finint
-/packages/ic-ledger-hash-of/         @dfinity/finint
-/packages/pocket-ic/                 @dfinity/pocket-ic
-/packages/ic-ethereum-types/         @dfinity/cross-chain-team
-/packages/ic-metrics-assert/         @dfinity/cross-chain-team
-/packages/ic-sha3/                   @dfinity/crypto-team
-/packages/ic-signature-verification/ @dfinity/crypto-team
-/packages/ic-vetkd-utils/            @dfinity/crypto-team
+/packages/icrc-cbor/                    @dfinity/finint @dfinity/cross-chain-team
+/packages/icrc-ledger-agent/            @dfinity/finint
+/packages/icrc-ledger-types/            @dfinity/finint
+/packages/ic-ledger-hash-of/            @dfinity/finint
+/packages/pocket-ic/                    @dfinity/pocket-ic
+/packages/ic-dummy-getrandom-for-wasm/  @dfinity/crypto-team
+/packages/ic-ed25519/                   @dfinity/crypto-team
+/packages/ic-ethereum-types/            @dfinity/cross-chain-team
+/packages/ic-metrics-assert/            @dfinity/cross-chain-team
+/packages/ic-secp256k1/                 @dfinity/crypto-team
+/packages/ic-sha3/                      @dfinity/crypto-team
+/packages/ic-signature-verification/    @dfinity/crypto-team
+/packages/ic-vetkd-utils/               @dfinity/crypto-team
 
 # [IC-OS]
 /ic-os/                                                                 @dfinity/node
@@ -250,6 +253,7 @@ go_deps.bzl               @dfinity/idx
 /rs/tests/execution/                                    @dfinity/execution
 /rs/tests/financial_integrations/                       @dfinity/finint
 /rs/tests/message_routing/                              @dfinity/ic-message-routing-owners
+/rs/tests/nested/                                       @dfinity/node
 /rs/tests/networking/                                   @dfinity/consensus
 /rs/tests/nns/                                          @dfinity/nns-team
 /rs/tests/node/                                         @dfinity/node

.github/actions/bazel-test-all/action.yaml

@@ -5,20 +5,12 @@ inputs:
     required: true
     default: 'test'
   BAZEL_TARGETS:
-    required: false
-    default: '//...'
-  BAZEL_CI_CONFIG:
-    required: false
-    default: '--config=ci'
-  BAZEL_EXTRA_ARGS:
-    required: false
-    default: ''
-  BAZEL_STARTUP_ARGS:
-    required: false
-    default: '--output_base=/var/tmp/bazel-output/'
+    required: true
   BUILDEVENT_APIKEY:
     required: false
-  SSH_PRIVATE_KEY:
+  SSH_PRIVATE_KEY_BACKUP_POD:
+    required: false
+  GPG_PASSPHRASE:
     required: false
 
 runs:
@@ -30,25 +22,25 @@ runs:
         run: |
           set +e # manual error handling to ensure we can run some post-build commands
 
-          if [ -n "$SSH_PRIVATE_KEY" ]; then
+          # freshly deployed k8s machines require ownership correctly set
+          if [ -e /cache ]; then
+            sudo chown -RL 1001:1001 /cache
+          fi
+
+          if [ -n "$SSH_PRIVATE_KEY_BACKUP_POD" ]; then
             # The following adds the SSH private key to the ssh-agent such that CI can SSH into the backup pod.
-            test -z "${SSH_AUTH_SOCK:-}" && { eval "$(ssh-agent -s)"; ssh-add - <<< "${SSH_PRIVATE_KEY}"; }
+            test -z "${SSH_AUTH_SOCK:-}" && { eval "$(ssh-agent -s)"; ssh-add - <<< "${SSH_PRIVATE_KEY_BACKUP_POD}"; }
             rm -rf ~/.ssh
             mkdir -p ~/.ssh
             chmod 0700 ~/.ssh
             echo -e "Host *\nUser github-runner\n" > ~/.ssh/config
           fi
 
-          # unset honeycomb api key but use it latter for exporter
+          # unset honeycomb api key but use it later for exporter
           # TODO: remove exporter when users can use superset
-          KEY=${BUILDEVENT_APIKEY:-""}
-          unset BUILDEVENT_APIKEY
-
-          ${GITHUB_WORKSPACE}/ci/bazel-scripts/main.sh
+          env -u BUILDEVENT_APIKEY ${GITHUB_WORKSPACE}/ci/bazel-scripts/main.sh
           BAZEL_EXIT_CODE="$?"
 
-          export BUILDEVENT_APIKEY="$KEY"
-
           if [ -n "$BUILDEVENT_APIKEY" ] && [ -f ./bazel-bep.pb ]; then
               # avoid output unless an error occurs during bes export. This ensures
               # only the (more relevant) output from the main bazel command is shown.
@@ -60,6 +52,11 @@ runs:
               rm "$exportout"
               echo "BEP events exported to honeycomb!"
           fi
+          if [ -n "$GPG_PASSPHRASE" ] && [ -f ./bazel-bep.pb ]; then
+              gpg --symmetric --cipher-algo AES256 -o bazel-bep.pb.gpg \
+                  --passphrase "$GPG_PASSPHRASE" --batch --yes bazel-bep.pb
+          fi
+          rm -f bazel-bep.pb
 
           # output node name to gihub step summary
           [ -n "${NODE_NAME:-}" ] && echo "Run on node: $NODE_NAME" >>$GITHUB_STEP_SUMMARY
@@ -68,13 +65,11 @@ runs:
         env:
           BAZEL_COMMAND: ${{ inputs.BAZEL_COMMAND }}
           BAZEL_TARGETS: ${{ inputs.BAZEL_TARGETS }}
-          BAZEL_CI_CONFIG: ${{ inputs.BAZEL_CI_CONFIG }}
-          BAZEL_EXTRA_ARGS: ${{ inputs.BAZEL_EXTRA_ARGS }}
-          BAZEL_STARTUP_ARGS: ${{ inputs.BAZEL_STARTUP_ARGS }}
           BRANCH_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           BUILDEVENT_APIKEY: ${{ inputs.BUILDEVENT_APIKEY }}
           CI_EVENT_NAME: ${{ github.event_name }}
           CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           CI_PULL_REQUEST_TARGET_BRANCH_NAME: ${{ github.event.pull_request.base.ref }}
           MERGE_BASE_SHA: ${{ github.event.pull_request.base.sha }}
-          SSH_PRIVATE_KEY: ${{ inputs.SSH_PRIVATE_KEY }}
+          SSH_PRIVATE_KEY_BACKUP_POD: ${{ inputs.SSH_PRIVATE_KEY_BACKUP_POD }}
+          GPG_PASSPHRASE: ${{ inputs.GPG_PASSPHRASE }}

.github/workflows-source/ci-main.yml

@@ -27,7 +27,6 @@ env:
   CI_EVENT_NAME: ${{ github.event_name }}
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   CI_RUN_ID: ${{ github.run_id }}
-  RUSTFLAGS: "--remap-path-prefix=${{ github.workspace }}=/ic"
   BUILDEVENT_DATASET: "github-ci-dfinity"
 
 anchors:
@@ -65,15 +64,6 @@ anchors:
     uses: actions/setup-python@v5
     with:
       python-version: '3.12'
-  bazel-upload: &bazel-upload
-    name: Upload bazel-targets
-    uses: actions/upload-artifact@v4
-    with:
-      name: bazel-targets
-      retention-days: 14
-      if-no-files-found: error
-      path: |
-        bazel-targets
   bazel-bep: &bazel-bep
     name: Upload bazel-bep
     # runs only if previous step succeeded or failed;
@@ -86,18 +76,18 @@ anchors:
       if-no-files-found: ignore
       compression-level: 9
       path: |
-        bazel-bep.pb
+        bazel-bep.pb.gpg
         profile.json
 
 jobs:
   bazel-test-all:
     name: Bazel Test All
     <<: *dind-large-setup
     runs-on:
-      group: ln1
+      group: zh1
       labels: dind-large
     env:
-      AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
+      CLOUD_CREDENTIALS_CONTENT: ${{ secrets.CLOUD_CREDENTIALS_CONTENT }}
       # Only run ci/bazel-scripts/diff.sh on PRs that are not labeled with "CI_ALL_BAZEL_TARGETS".
       OVERRIDE_DIDC_CHECK: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_DIDC_CHECK') }}
       CI_OVERRIDE_BUF_BREAKING: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_BUF_BREAKING') }}
@@ -147,29 +137,10 @@ jobs:
         id: bazel-test-all
         uses: ./.github/actions/bazel-test-all/
         with:
-          BAZEL_COMMAND: "test"
-          BAZEL_TARGETS: "//..."
-          BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
-          # check if PR title contains release and set timeout filters accordingly
-          BAZEL_EXTRA_ARGS: ${{ env.BAZEL_EXTRA_ARGS }}
-          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
-      - <<: *bazel-bep
-      - <<: *bazel-upload
-
-  bazel-build-all-config-check:
-    <<: *dind-large-setup
-    name: Bazel Build All Config Check
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI_BUILD_CHECK') }}
-    steps:
-      - <<: *checkout
-      - <<: *before-script
-      - name: Run bazel build --config=check //rs/...
-        id: bazel-build-config-check
-        uses: ./.github/actions/bazel-test-all/
-        with:
-          BAZEL_COMMAND: "build"
-          BAZEL_TARGETS: "//rs/..."
-          BAZEL_CI_CONFIG: "--config=check --config=ci --keep_going"
+          BAZEL_COMMAND: test --config=ci ${{ env.BAZEL_EXTRA_ARGS }}
+          BAZEL_TARGETS: //...
+          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_TOKEN }}
+          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
       - <<: *bazel-bep
 
   bazel-test-macos-intel:
@@ -195,19 +166,20 @@ jobs:
         id: bazel-test-darwin-x86-64
         uses:  ./.github/actions/bazel-test-all/
         env:
-          AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
+          CLOUD_CREDENTIALS_CONTENT: ${{ secrets.CLOUD_CREDENTIALS_CONTENT }}
         with:
-          BAZEL_CI_CONFIG: "--config=ci --config macos_ci"
-          BAZEL_COMMAND: test
-          BAZEL_EXTRA_ARGS: '--test_tag_filters=test_macos'
-          BAZEL_STARTUP_ARGS: "--output_base /var/tmp/bazel-output/${{ github.run_id }}"
-          BAZEL_TARGETS: "//rs/... //publish/binaries/..."
+          BAZEL_COMMAND: >-
+            test --config=ci --config=macos_ci
+            --test_tag_filters=test_macos
+          BAZEL_TARGETS: //rs/... //publish/binaries/...
+          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
       - <<: *bazel-bep
       - name: Purge Bazel Output
         if: always()
         shell: bash
         run: |
-          sudo rm -rf /private/var/tmp/bazel-output
+          # Clean up the output base for the next run
+          sudo rm -rf /var/tmp/bazel-output
 
   bazel-build-fuzzers:
     name: Bazel Build Fuzzers
@@ -218,9 +190,10 @@ jobs:
         id: bazel-build-fuzzers
         uses:  ./.github/actions/bazel-test-all/
         with:
-          BAZEL_COMMAND: "build"
-          BAZEL_TARGETS: "//rs/..."
-          BAZEL_EXTRA_ARGS: "--keep_going --config=fuzzing --build_tag_filters=libfuzzer"
+          BAZEL_COMMAND: >-
+            build --config=ci --keep_going --config=fuzzing --build_tag_filters=libfuzzer
+          BAZEL_TARGETS: //rs/...
+          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
       - <<: *bazel-bep
 
   bazel-build-fuzzers-afl:
@@ -232,9 +205,9 @@ jobs:
         id: bazel-build-fuzzers-afl
         uses:  ./.github/actions/bazel-test-all/
         with:
-          BAZEL_COMMAND: "build"
-          BAZEL_TARGETS: "//rs/..."
-          BAZEL_EXTRA_ARGS: "--keep_going --config=afl"
+          BAZEL_COMMAND: build --config=ci --keep_going --config=afl
+          BAZEL_TARGETS: //rs/...
+          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
       - <<: *bazel-bep
 
   python-ci-tests:
@@ -284,7 +257,8 @@ jobs:
           "$CI_PROJECT_DIR"/ci/scripts/run-build-ic.sh
           rm -rf "/cache/job/${CI_JOB_NAME}/${CI_RUN_ID}"
         env:
-          BAZEL_COMMAND: "build"
+          BAZEL_COMMAND: build --config=ci
+          BAZEL_TARGETS: //...
           MERGE_BASE_SHA: ${{ github.event.pull_request.base.sha }}
           BRANCH_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
@@ -301,6 +275,8 @@ jobs:
     name: Build Determinism
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    # NOTE: this expects "build-ic" to have built the same set of targets
+    # as "bazel-test-all"
     needs: [build-ic, bazel-test-all]
     strategy:
       matrix:
@@ -327,20 +303,14 @@ jobs:
             SETUPOS_FLAG: "true"
     steps:
       - <<: *checkout
-      - name: Download bazel-targets [bazel-test-all]
-        uses: actions/download-artifact@v4
-        with:
-          name: bazel-targets
       - name: Download build-ic.tar [build-ic]
         uses: actions/download-artifact@v4
         with:
           name: build-ic
       - name: Build Determinism Test
         id: build-determinism
-        shell: bash
         run: |
-          set -eExuo pipefail
-          sudo apt update && sudo apt install -y curl
+          tar -xf build-ic.tar # build-determinism.sh expects ./build-ic/
           "$CI_PROJECT_DIR"/ci/scripts/build-determinism.sh
         env:
           TARGET: ${{ matrix.TARGET }}

.github/workflows-source/ci-pr-only.yml

@@ -64,17 +64,6 @@ jobs:
           set -euo pipefail
           cd "${GITHUB_WORKSPACE}"/bin
           ./build-all-fuzzers.sh --zip
-      - name: Upload bazel-bep
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ github.job }}-bep
-          retention-days: 14
-          if-no-files-found: ignore
-          compression-level: 9
-          path: |
-            bazel-bep.pb
-            profile.json
 
   lock-generate:
     name: Lock Generate