schedule-daily.yml

name: Schedule Daily
on:
  schedule:
    - cron: "0 1 * * *"
  workflow_dispatch:
env:
  BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/"
  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
  CI_COMMIT_SHA: ${{ github.sha }}
  CI_JOB_NAME: ${{ github.job }}
  CI_PROJECT_DIR: ${{ github.workspace }}
  CI_RUN_ID: ${{ github.run_id }}
  BUILDEVENT_DATASET: "github-ci-dfinity"
jobs:
  bazel-test-bare-metal:
    name: Bazel Test Bare Metal
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 120
    runs-on:
      group: zh1
      labels: dind-large
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Before script
        id: before-script
        shell: bash
        run: |
          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
      - name: Run Bazel Launch Bare Metal
        shell: bash
        run: |
          echo "$ZH2_DLL01_CSV_SECRETS" > file1
          echo "$ZH2_FILE_SHARE_KEY" > file2 && chmod 400 file2
          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"

          # Run bare metal installation test
          # shellcheck disable=SC2046,SC2086
          bazel ${BAZEL_STARTUP_ARGS} run ${BAZEL_CI_CONFIG} \
            //ic-os/setupos/envs/dev:launch_bare_metal -- \
              --config_path "$(realpath  ./ic-os/dev-tools/bare_metal_deployment/zh2-dll01.yaml)" \
              --csv_filename "$(realpath file1)" \
              --file_share_ssh_key "$(realpath file2)" \
              --inject_image_pub_key "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK3gjE/2K5nxIBbk3ohgs8J5LW+XiObwA+kGtSaF5+4c" \
              --file_share_username ci_interim \
              --hsm \
              --ci_mode

          # Run bare metal node performance benchmarks
          # shellcheck disable=SC2046,SC2086
          bazel ${BAZEL_STARTUP_ARGS} run ${BAZEL_CI_CONFIG} \
            //ic-os/setupos/envs/dev:launch_bare_metal -- \
              --config_path "$(realpath  ./ic-os/dev-tools/bare_metal_deployment/zh2-dll01.yaml)" \
              --csv_filename "$(realpath file1)" \
              --file_share_ssh_key "$(realpath file2)" \
              --inject_image_pub_key "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK3gjE/2K5nxIBbk3ohgs8J5LW+XiObwA+kGtSaF5+4c" \
              --file_share_username ci_interim \
              --ci_mode \
              --benchmark
          bazel clean
        env:
          BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/"
          BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
          ZH2_DLL01_CSV_SECRETS: "${{ secrets.ZH2_DLL01_CSV_SECRETS }}"
          ZH2_FILE_SHARE_KEY: "${{ secrets.ZH2_FILE_SHARE_KEY }}"
  fi-tests-nightly:
    name: Bazel Test FI Nightly
    runs-on:
      group: zh1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 720 # 12 hours
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Before script
        id: before-script
        shell: bash
        run: |
          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
      - name: Run FI Tests Nightly
        id: bazel-test-all
        uses: ./.github/actions/bazel-test-all/
        with:
          BAZEL_COMMAND: "test"
          BAZEL_TARGETS: "//rs/ledger_suite/..."
          BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
          BAZEL_EXTRA_ARGS: "--keep_going --test_tag_filters=fi_tests_nightly --test_env=SSH_AUTH_SOCK --test_timeout=43200"
          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_TOKEN }}
          SSH_PRIVATE_KEY_BACKUP_POD: ${{ secrets.SSH_PRIVATE_KEY_BACKUP_POD }}
          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
      - name: Upload bazel-bep
        # runs only if previous step succeeded or failed;
        # we avoid collecting artifacts of jobs that were cancelled
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-bep
          retention-days: 14
          if-no-files-found: ignore
          compression-level: 9
          path: |
            bazel-bep.pb.gpg
            profile.json
  nns-tests-nightly:
    name: Bazel Test NNS Nightly
    runs-on:
      group: zh1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 20
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Before script
        id: before-script
        shell: bash
        run: |
          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
      - name: Run NNS Tests Nightly
        id: bazel-test-all
        uses: ./.github/actions/bazel-test-all/
        with:
          BAZEL_COMMAND: "test"
          BAZEL_TARGETS: "//rs/nns/..."
          BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
          BAZEL_EXTRA_ARGS: "--keep_going --test_tag_filters=nns_tests_nightly --test_env=SSH_AUTH_SOCK --test_env=NNS_CANISTER_UPGRADE_SEQUENCE=all"
          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_TOKEN }}
          SSH_PRIVATE_KEY_BACKUP_POD: ${{ secrets.SSH_PRIVATE_KEY_BACKUP_POD }}
          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
      - name: Upload bazel-bep
        # runs only if previous step succeeded or failed;
        # we avoid collecting artifacts of jobs that were cancelled
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-bep
          retention-days: 14
          if-no-files-found: ignore
          compression-level: 9
          path: |
            bazel-bep.pb.gpg
            profile.json
  system-tests-benchmarks-nightly:
    name: Bazel System Test Benchmarks
    runs-on:
      group: zh1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 480
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Before script
        id: before-script
        shell: bash
        run: |
          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
      - name: Set Benchmark Targets
        shell: bash
        run: |
          set -xeuo pipefail
          echo "BENCHMARK_TARGETS=$(bazel query 'attr(tags, system_test_benchmark, //rs/...)' | grep -v head_nns | tr '\n' ' ')" >> $GITHUB_ENV
      - name: Test System Test Benchmarks
        id: bazel-system-test-benchmarks
        uses: ./.github/actions/bazel-test-all/
        with:
          BAZEL_COMMAND: "test"
          BAZEL_TARGETS: ${{ env.BENCHMARK_TARGETS }}
          BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
          # note: there's just one performance cluster, so the job can't be parallelized
          BAZEL_EXTRA_ARGS: "--test_tag_filters=system_test_benchmark --//bazel:enable_upload_perf_systest_results=True --keep_going --jobs 1"
          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_TOKEN }}
          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
      - name: Upload bazel-bep
        # runs only if previous step succeeded or failed;
        # we avoid collecting artifacts of jobs that were cancelled
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-bep
          retention-days: 14
          if-no-files-found: ignore
          compression-level: 9
          path: |
            bazel-bep.pb.gpg
            profile.json
      - name: Post Slack Notification
        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
        if: failure()
        with:
          channel-id: eng-crypto-alerts
          slack-message: "${{ github.job }} failed :disappointed: - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Run#${{github.run_id}}>"
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_API_TOKEN }}
  dependency-scan-nightly:
    name: Dependency Scan Nightly
    runs-on:
      group: zh1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 60
    permissions:
      actions: write
    env:
      SHELL_WRAPPER: "/usr/bin/time"
      CARGO_WASMPACK_VERSION: "0.12.1"
      DEFAULT_NODE_VERSION: "20"
      CI_PROJECT_PATH: ${{ github.repository }}
      CI_PIPELINE_ID: ${{ github.run_id }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
      SLACK_PSEC_BOT_OAUTH_TOKEN: ${{ secrets.SLACK_PSEC_BOT_OAUTH_TOKEN }}
      GITHUB_REF: ${{ github.ref }}
      REPO_NAME: ${{ github.repository }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Setup environment deps
        id: setup-environment-deps
        shell: bash
        run: |
          # Ignore externally-managed-environment pip error, install packages system-wide.
          PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --ignore-installed -r requirements.txt
          cargo install wasm-pack --version "${CARGO_WASMPACK_VERSION}"
          source "${NVM_DIR}/nvm.sh"
          nvm use ${DEFAULT_NODE_VERSION}
          node --version
          npm --version
      - name: Run Dependency Scan Nightly
        id: dependency-scan-nightly
        shell: bash
        run: |
          set -euo pipefail
          export PYTHONPATH=$PWD/ci/src:$PWD/ci/src/dependencies
          cd ci/src/dependencies/
          $SHELL_WRAPPER python3 job/bazel_rust_ic_scanner_periodic_job.py
          $SHELL_WRAPPER python3 job/npm_scanner_periodic_job.py
          $SHELL_WRAPPER python3 job/bazel_trivy_container_ic_scanner_periodic_job.py
  # CI job is also executed in PR on use of 'CI_COVERAGE' label
  bazel-test-coverage:
    name: Bazel Test Coverage
    runs-on:
      group: zh1
      labels: dind-large
    container:
      image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d
      options: >-
        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
    timeout-minutes: 120
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Run Bazel Test Coverage
        shell: bash
        run: |
          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
          ./ci/scripts/bazel-coverage.sh
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}