diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7b78690da44..b3fc551e6d5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "image": "ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273", + "image": "ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3", "remoteUser": "ubuntu", "privileged": true, "runArgs": [ @@ -16,7 +16,8 @@ "workspaceFolder": "/ic", "initializeCommand": "mkdir -p ~/.aws ~/.ssh ~/.cache/cargo ~/.local/share/fish && touch ~/.zsh_history ~/.bash_history", "containerEnv": { - "CARGO_TARGET_DIR": "/home/ubuntu/.cache/cargo" + "CARGO_TARGET_DIR": "/home/ubuntu/.cache/cargo", + "USER": "${localEnv:USER}" }, "mounts": [ { diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6bfa4dc3463..54a6a0e0288 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -94,7 +94,7 @@ go_deps.bzl @dfinity/idx /rs/bitcoin/ @dfinity/ic-interface-owners /rs/bitcoin/adapter/ @dfinity/networking /rs/bitcoin/ckbtc/ @dfinity/cross-chain-team -/rs/bitcoin/mock/ @dfinity/finint +/rs/bitcoin/mock/ @dfinity/cross-chain-team /rs/bitcoin/client/ @dfinity/networking /rs/bitcoin/consensus/ @dfinity/execution @dfinity/consensus /rs/bitcoin/kyt/ @dfinity/cross-chain-team diff --git a/.github/actions/bazel-test-all/action.yaml b/.github/actions/bazel-test-all/action.yaml index f7273739779..85d5e2f62bb 100644 --- a/.github/actions/bazel-test-all/action.yaml +++ b/.github/actions/bazel-test-all/action.yaml @@ -68,6 +68,7 @@ runs: BAZEL_CI_CONFIG: ${{ inputs.BAZEL_CI_CONFIG }} BAZEL_EXTRA_ARGS: ${{ inputs.BAZEL_EXTRA_ARGS }} BAZEL_STARTUP_ARGS: ${{ inputs.BAZEL_STARTUP_ARGS }} + CI_EVENT_NAME: ${{ github.event_name }} CI_PULL_REQUEST_TARGET_BRANCH_NAME: ${{ github.event.pull_request.base.ref }} MERGE_BASE_SHA: ${{ github.event.pull_request.base.sha }} BRANCH_HEAD_SHA: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows-source/ci-main.yml b/.github/workflows-source/ci-main.yml index 0624b743927..b99f14cf3f5 100644 --- a/.github/workflows-source/ci-main.yml +++ b/.github/workflows-source/ci-main.yml @@ -24,7 +24,6 @@ env: CI_COMMIT_SHA: ${{ github.sha }} CI_JOB_NAME: ${{ github.job }} CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - CI_PIPELINE_SOURCE: ${{ github.event_name }} CI_PROJECT_DIR: ${{ github.workspace }} CI_EVENT_NAME: ${{ github.event_name }} BRANCH_NAME: ${{ github.head_ref || github.ref_name }} @@ -34,7 +33,7 @@ env: anchors: image: &image - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 dind-large-setup: &dind-large-setup runs-on: labels: dind-large diff --git a/.github/workflows-source/ci-pr-only.yml b/.github/workflows-source/ci-pr-only.yml index b3f280aa35a..4f0f439016e 100644 --- a/.github/workflows-source/ci-pr-only.yml +++ b/.github/workflows-source/ci-pr-only.yml @@ -11,14 +11,13 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref }} cancel-in-progress: true env: - CI_PIPELINE_SOURCE: ${{ github.event_name }} CI_PROJECT_DIR: ${{ github.workspace }} MERGE_BRANCH: ${{ github.event.pull_request.base.ref }} ORG: ${{ github.repository_owner }} anchors: image: &image - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 dind-small-setup: &dind-small-setup timeout-minutes: 30 runs-on: @@ -81,6 +80,8 @@ jobs: name: Lock Generate <<: *dind-small-setup <<: *skip-merge-group + env: + CI_EVENT_NAME: ${{ github.event_name }} steps: - name: Create GitHub App Token uses: actions/create-github-app-token@v1 diff --git a/.github/workflows-source/release-testing.yml b/.github/workflows-source/release-testing.yml index 903f22d9304..8611ef4d72e 100644 --- a/.github/workflows-source/release-testing.yml +++ b/.github/workflows-source/release-testing.yml @@ -16,7 +16,6 @@ env: CI_COMMIT_SHA: ${{ github.sha }} CI_JOB_NAME: ${{ github.job }} CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - CI_PIPELINE_SOURCE: ${{ github.event_name }} CI_PROJECT_DIR: ${{ github.workspace }} BRANCH_NAME: ${{ github.event.workflow_run.head_branch || github.ref_name }} CI_RUN_ID: ${{ github.run_id }} @@ -25,7 +24,7 @@ env: anchors: image: &image - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 dind-large-setup: &dind-large-setup runs-on: group: zh1 diff --git a/.github/workflows-source/schedule-daily.yml b/.github/workflows-source/schedule-daily.yml index 8311e68fbca..8026c4b14e4 100644 --- a/.github/workflows-source/schedule-daily.yml +++ b/.github/workflows-source/schedule-daily.yml @@ -11,14 +11,13 @@ env: CI_COMMIT_SHA: ${{ github.sha }} CI_JOB_NAME: ${{ github.job }} CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - CI_PIPELINE_SOURCE: ${{ github.event_name }} CI_PROJECT_DIR: ${{ github.workspace }} CI_RUN_ID: ${{ github.run_id }} BUILDEVENT_DATASET: "github-ci-dfinity" anchors: image: &image - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 dind-large-setup: &dind-large-setup runs-on: group: zh1 diff --git a/.github/workflows-source/schedule-hourly.yml b/.github/workflows-source/schedule-hourly.yml index c5a3091434d..d8c123d4986 100644 --- a/.github/workflows-source/schedule-hourly.yml +++ b/.github/workflows-source/schedule-hourly.yml @@ -10,7 +10,6 @@ env: CI_COMMIT_SHA: ${{ github.sha }} CI_JOB_NAME: ${{ github.job }} CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - CI_PIPELINE_SOURCE: ${{ github.event_name }} CI_PROJECT_DIR: ${{ github.workspace }} CI_RUN_ID: ${{ github.run_id }} RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic" @@ -18,7 +17,7 @@ env: anchors: image: &image - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 dind-large-setup: &dind-large-setup runs-on: labels: dind-large diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 41e3956cf8b..c4c85e35889 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -21,7 +21,6 @@ env: CI_COMMIT_SHA: ${{ github.sha }} CI_JOB_NAME: ${{ github.job }} CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - CI_PIPELINE_SOURCE: ${{ github.event_name }} CI_PROJECT_DIR: ${{ github.workspace }} CI_EVENT_NAME: ${{ github.event_name }} BRANCH_NAME: ${{ github.head_ref || github.ref_name }} @@ -32,7 +31,7 @@ jobs: bazel-test-all: name: Bazel Test All container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info timeout-minutes: 90 @@ -131,7 +130,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info timeout-minutes: 90 @@ -225,7 +224,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info timeout-minutes: 90 @@ -260,7 +259,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info timeout-minutes: 90 @@ -295,7 +294,7 @@ jobs: runs-on: labels: dind-small container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 timeout-minutes: 30 steps: - name: Checkout @@ -325,7 +324,7 @@ jobs: build-ic: name: Build IC container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info timeout-minutes: 90 @@ -428,7 +427,7 @@ jobs: cargo-clippy-linux: name: Cargo Clippy Linux container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 timeout-minutes: 30 runs-on: group: ch1 @@ -465,7 +464,7 @@ jobs: cargo-build-release-linux: name: Cargo Build Release Linux container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 timeout-minutes: 30 runs-on: group: ch1 diff --git a/.github/workflows/ci-pr-only.yml b/.github/workflows/ci-pr-only.yml index 7f68ca8856c..b5762b2af44 100644 --- a/.github/workflows/ci-pr-only.yml +++ b/.github/workflows/ci-pr-only.yml @@ -10,7 +10,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref }} cancel-in-progress: true env: - CI_PIPELINE_SOURCE: ${{ github.event_name }} CI_PROJECT_DIR: ${{ github.workspace }} MERGE_BRANCH: ${{ github.event.pull_request.base.ref }} ORG: ${{ github.repository_owner }} @@ -21,7 +20,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME if: ${{ github.event_name != 'merge_group' }} @@ -62,10 +61,12 @@ jobs: runs-on: labels: dind-small container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME if: ${{ github.event_name != 'merge_group' }} + env: + CI_EVENT_NAME: ${{ github.event_name }} steps: - name: Create GitHub App Token uses: actions/create-github-app-token@v1 @@ -101,7 +102,7 @@ jobs: runs-on: labels: dind-small container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME if: ${{ github.event_name != 'merge_group' }} @@ -149,7 +150,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME if: contains(github.event.pull_request.labels.*.name, 'CI_COVERAGE') diff --git a/.github/workflows/release-testing.yml b/.github/workflows/release-testing.yml index 7faa7d9365f..a685050eaed 100644 --- a/.github/workflows/release-testing.yml +++ b/.github/workflows/release-testing.yml @@ -13,7 +13,6 @@ env: CI_COMMIT_SHA: ${{ github.sha }} CI_JOB_NAME: ${{ github.job }} CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - CI_PIPELINE_SOURCE: ${{ github.event_name }} CI_PROJECT_DIR: ${{ github.workspace }} BRANCH_NAME: ${{ github.event.workflow_run.head_branch || github.ref_name }} CI_RUN_ID: ${{ github.run_id }} @@ -30,7 +29,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 180 # 3 hours @@ -73,7 +72,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 180 # 3 hours @@ -116,7 +115,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 90 @@ -159,7 +158,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 60 @@ -207,7 +206,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 180 # 3 hours @@ -235,7 +234,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 180 # 3 hours diff --git a/.github/workflows/rosetta-release.yml b/.github/workflows/rosetta-release.yml index 0b8c1336c8f..02bc9f13215 100644 --- a/.github/workflows/rosetta-release.yml +++ b/.github/workflows/rosetta-release.yml @@ -22,7 +22,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 steps: - name: Checkout uses: actions/checkout@v4 @@ -89,5 +89,5 @@ jobs: for tag in "v${ICRC_ROSETTA_RELEASE_VERSION}" latest; do bazel run --stamp --embed_label="$tag" //rs/rosetta-api/icrc1:push_ic_icrc_rosetta_image done - git tag "icrc-rosetta-release-$ICRC_ROSETTA_RELEASE_VERSION" "${{ github.sha }}" - git push origin "icrc-rosetta-release-$ICRC_ROSETTA_RELEASE_VERSION" + git tag "rosetta-icrc-release-$ICRC_ROSETTA_RELEASE_VERSION" "${{ github.sha }}" + git push origin "rosetta-icrc-release-$ICRC_ROSETTA_RELEASE_VERSION" diff --git a/.github/workflows/schedule-daily.yml b/.github/workflows/schedule-daily.yml index 16c2cab989b..8ba8c49678e 100644 --- a/.github/workflows/schedule-daily.yml +++ b/.github/workflows/schedule-daily.yml @@ -9,7 +9,6 @@ env: CI_COMMIT_SHA: ${{ github.sha }} CI_JOB_NAME: ${{ github.job }} CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - CI_PIPELINE_SOURCE: ${{ github.event_name }} CI_PROJECT_DIR: ${{ github.workspace }} CI_RUN_ID: ${{ github.run_id }} BUILDEVENT_DATASET: "github-ci-dfinity" @@ -17,7 +16,7 @@ jobs: bazel-test-bare-metal: name: Bazel Test Bare Metal container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 120 @@ -75,7 +74,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 720 # 12 hours @@ -117,7 +116,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 20 @@ -159,7 +158,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 480 @@ -206,7 +205,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 60 @@ -258,7 +257,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 120 diff --git a/.github/workflows/schedule-hourly.yml b/.github/workflows/schedule-hourly.yml index 3cc86817af1..f313e5c6bc5 100644 --- a/.github/workflows/schedule-hourly.yml +++ b/.github/workflows/schedule-hourly.yml @@ -8,7 +8,6 @@ env: CI_COMMIT_SHA: ${{ github.sha }} CI_JOB_NAME: ${{ github.job }} CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - CI_PIPELINE_SOURCE: ${{ github.event_name }} CI_PROJECT_DIR: ${{ github.workspace }} CI_RUN_ID: ${{ github.run_id }} RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic" @@ -19,7 +18,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 120 @@ -56,7 +55,7 @@ jobs: bazel-system-test-hourly: name: Bazel System Tests Hourly container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 120 @@ -99,7 +98,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 120 diff --git a/.github/workflows/schedule-rust-bench.yml b/.github/workflows/schedule-rust-bench.yml index fbd4035e9cc..a16e5c3bc36 100644 --- a/.github/workflows/schedule-rust-bench.yml +++ b/.github/workflows/schedule-rust-bench.yml @@ -11,7 +11,6 @@ env: CI_COMMIT_SHA: ${{ github.sha }} CI_JOB_NAME: ${{ github.job }} CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - CI_PIPELINE_SOURCE: ${{ github.event_name }} CI_PROJECT_DIR: ${{ github.workspace }} CI_RUN_ID: ${{ github.run_id }} BUILDEVENT_DATASET: "github-ci-dfinity" @@ -24,7 +23,7 @@ jobs: # see linux-x86-64 runner group labels: rust-benchmarks container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 # running on bare metal machine using ubuntu user options: --user ubuntu -v /cache:/cache timeout-minutes: 720 # 12 hours diff --git a/.github/workflows/schedule-weekly.yml b/.github/workflows/schedule-weekly.yml index 4d3e5283ce3..9c19416ec36 100644 --- a/.github/workflows/schedule-weekly.yml +++ b/.github/workflows/schedule-weekly.yml @@ -14,7 +14,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME -v /cache:/cache diff --git a/.github/workflows/sync-public-with-private.yml b/.github/workflows/sync-public-with-private.yml index 9996dc8c3a7..5bbb48c2bc8 100644 --- a/.github/workflows/sync-public-with-private.yml +++ b/.github/workflows/sync-public-with-private.yml @@ -13,17 +13,25 @@ jobs: image: ghcr.io/dfinity/minimal-runner-image:0.1 if: ${{ github.repository != 'dfinity/ic' }} steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: 'master-private' - token: ${{ secrets.PUSH_TO_IC_PRIVATE }} - - - name: Sync changes from public ic - run: | - git config --global user.name "IDX GitHub Automation" - git config --global user.email "idx@dfinity.org" - - git remote add public https://github.com/dfinity/ic.git - git fetch public master - git push origin public/master:master-private + - name: Create GitHub App Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + # The app used for this is actually PR_AUTOMATION_BOT_PRIVATE, but because the env + # vars need to be the same across both repos, we use the same names as the public one + app-id: ${{ vars.PR_AUTOMATION_BOT_PUBLIC_APP_ID }} + private-key: ${{ secrets.PR_AUTOMATION_BOT_PUBLIC_PRIVATE_KEY }} + - name: Checkout + uses: actions/checkout@v4 + with: + ref: 'master-private' + token: ${{ steps.app-token.outputs.token }} + + - name: Sync changes from public ic + run: | + git config --global user.name "IDX GitHub Automation" + git config --global user.email "idx@dfinity.org" + + git remote add public https://github.com/dfinity/ic.git + git fetch public master + git push origin public/master:master-private diff --git a/.github/workflows/system-tests-k8s.yml b/.github/workflows/system-tests-k8s.yml index 3441f547092..9b63accc484 100644 --- a/.github/workflows/system-tests-k8s.yml +++ b/.github/workflows/system-tests-k8s.yml @@ -49,7 +49,7 @@ jobs: group: ln1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME -e KUBECONFIG --privileged --cgroupns host @@ -104,7 +104,7 @@ jobs: group: ln1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:3cc80d1a2d5789b69f6a7f170280bb3763a54acce299430903f59718f904f273 + image: ghcr.io/dfinity/ic-build@sha256:6a548dd4cfee96260b6938fddaa74abc86fc0df5c407d4eef135694b64c288e3 options: >- -e NODE_NAME -e KUBECONFIG --privileged --cgroupns host diff --git a/Cargo.Bazel.Fuzzing.json.lock b/Cargo.Bazel.Fuzzing.json.lock index 6a4abb6e55e..39bd2bcf3d2 100644 --- a/Cargo.Bazel.Fuzzing.json.lock +++ b/Cargo.Bazel.Fuzzing.json.lock @@ -1,5 +1,5 @@ { - "checksum": "1771e8a5f1fdb0c8cf153411c37a9a8aab8ff03252dbb22817a79363ad197c9b", + "checksum": "3fbe6f0bc1150dc7e649e3be6d14ab3560751213130f5f2404d9e38092e2d002", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -19028,6 +19028,10 @@ "id": "lru 0.7.8", "target": "lru" }, + { + "id": "macaddr 1.0.1", + "target": "macaddr" + }, { "id": "mach2 0.4.2", "target": "mach2" @@ -40066,7 +40070,13 @@ "commitish": { "Rev": "4d952c8f1dca79de855af892b444d7112567b58d" }, - "strip_prefix": "lmdb-sys" + "strip_prefix": "lmdb-sys", + "patch_args": [ + "-p1" + ], + "patches": [ + "@@//bazel:lmdb_rkv_sys.patch" + ] } }, "targets": [ @@ -40119,12 +40129,6 @@ ], "selects": {} }, - "extra_deps": { - "common": [ - "@lmdb//:lmdb" - ], - "selects": {} - }, "edition": "2015", "version": "0.11.99" }, @@ -40134,6 +40138,7 @@ ], "data": { "common": [ + "@lmdb//:liblmdb", "@lmdb//:lmdb.h" ], "selects": {} @@ -40157,7 +40162,7 @@ "build_script_env": { "common": { "LMDB_H_PATH": "$(location @lmdb//:lmdb.h)", - "LMDB_NO_BUILD": "1" + "LMDB_OVERRIDE": "$(location @lmdb//:liblmdb)" }, "selects": {} } @@ -40974,6 +40979,52 @@ ], "license_file": null }, + "macaddr 1.0.1": { + "name": "macaddr", + "version": "1.0.1", + "package_url": "https://github.com/svartalf/rust-macaddr", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/macaddr/1.0.1/download", + "sha256": "baee0bbc17ce759db233beb01648088061bf678383130602a298e6998eedb2d8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "macaddr", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "macaddr", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "edition": "2018", + "version": "1.0.1" + }, + "license": "Apache-2.0 OR MIT", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, "mach2 0.4.2": { "name": "mach2", "version": "0.4.2", @@ -87754,6 +87805,7 @@ "lmdb-rkv-sys 0.11.99", "local-ip-address 0.5.6", "lru 0.7.8", + "macaddr 1.0.1", "mach2 0.4.2", "maplit 1.0.2", "maxminddb 0.24.0", diff --git a/Cargo.Bazel.Fuzzing.toml.lock b/Cargo.Bazel.Fuzzing.toml.lock index 16442ffcaa4..75d9febc2e9 100644 --- a/Cargo.Bazel.Fuzzing.toml.lock +++ b/Cargo.Bazel.Fuzzing.toml.lock @@ -3140,6 +3140,7 @@ dependencies = [ "lmdb-rkv-sys", "local-ip-address", "lru", + "macaddr", "mach2", "maplit", "maxminddb", @@ -6686,6 +6687,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "macaddr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baee0bbc17ce759db233beb01648088061bf678383130602a298e6998eedb2d8" + [[package]] name = "mach2" version = "0.4.2" diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index d79b92c2d18..1aaef252187 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "8a0463c997697666d544fe6b2e5aae9ebf90ceb2daaaaca94f6e0664819e9af0", + "checksum": "a3824c6dae08a250d31e5907e2eff62575b38fd163dbb7e1c0f0ba342d66dc29", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -18856,6 +18856,10 @@ "id": "lru 0.7.8", "target": "lru" }, + { + "id": "macaddr 1.0.1", + "target": "macaddr" + }, { "id": "mach2 0.4.2", "target": "mach2" @@ -39903,7 +39907,13 @@ "commitish": { "Rev": "4d952c8f1dca79de855af892b444d7112567b58d" }, - "strip_prefix": "lmdb-sys" + "strip_prefix": "lmdb-sys", + "patch_args": [ + "-p1" + ], + "patches": [ + "@@//bazel:lmdb_rkv_sys.patch" + ] } }, "targets": [ @@ -39956,12 +39966,6 @@ ], "selects": {} }, - "extra_deps": { - "common": [ - "@lmdb//:lmdb" - ], - "selects": {} - }, "edition": "2015", "version": "0.11.99" }, @@ -39971,6 +39975,7 @@ ], "data": { "common": [ + "@lmdb//:liblmdb", "@lmdb//:lmdb.h" ], "selects": {} @@ -39994,7 +39999,7 @@ "build_script_env": { "common": { "LMDB_H_PATH": "$(location @lmdb//:lmdb.h)", - "LMDB_NO_BUILD": "1" + "LMDB_OVERRIDE": "$(location @lmdb//:liblmdb)" }, "selects": {} } @@ -40815,6 +40820,52 @@ ], "license_file": null }, + "macaddr 1.0.1": { + "name": "macaddr", + "version": "1.0.1", + "package_url": "https://github.com/svartalf/rust-macaddr", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/macaddr/1.0.1/download", + "sha256": "baee0bbc17ce759db233beb01648088061bf678383130602a298e6998eedb2d8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "macaddr", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "macaddr", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "edition": "2018", + "version": "1.0.1" + }, + "license": "Apache-2.0 OR MIT", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, "mach2 0.4.2": { "name": "mach2", "version": "0.4.2", @@ -87634,6 +87685,7 @@ "lmdb-rkv-sys 0.11.99", "local-ip-address 0.5.6", "lru 0.7.8", + "macaddr 1.0.1", "mach2 0.4.2", "maplit 1.0.2", "maxminddb 0.24.0", diff --git a/Cargo.Bazel.toml.lock b/Cargo.Bazel.toml.lock index 46261e373f7..556b8d5bc51 100644 --- a/Cargo.Bazel.toml.lock +++ b/Cargo.Bazel.toml.lock @@ -3129,6 +3129,7 @@ dependencies = [ "lmdb-rkv-sys", "local-ip-address", "lru", + "macaddr", "mach2", "maplit", "maxminddb", @@ -6677,6 +6678,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "macaddr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baee0bbc17ce759db233beb01648088061bf678383130602a298e6998eedb2d8" + [[package]] name = "mach2" version = "0.4.2" diff --git a/Cargo.lock b/Cargo.lock index cad39cd54bc..c5c1eb3fde9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -362,24 +362,6 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" -[[package]] -name = "anonymization-backend" -version = "0.1.0" -dependencies = [ - "anonymization-interface", - "anyhow", - "async-trait", - "candid", - "ic-cdk 0.16.0", - "ic-cdk-timers", - "ic-nns-constants", - "ic-stable-structures", - "lazy_static", - "prometheus", - "serde", - "thiserror 2.0.3", -] - [[package]] name = "anonymization-client" version = "0.1.0" @@ -405,6 +387,24 @@ dependencies = [ "serde_bytes", ] +[[package]] +name = "anonymization_backend" +version = "0.1.0" +dependencies = [ + "anonymization-interface", + "anyhow", + "async-trait", + "candid", + "ic-cdk 0.16.0", + "ic-cdk-timers", + "ic-nns-constants", + "ic-stable-structures", + "lazy_static", + "prometheus", + "serde", + "thiserror 2.0.3", +] + [[package]] name = "anstream" version = "0.6.15" @@ -1822,7 +1822,7 @@ name = "canister_client" version = "0.9.0" dependencies = [ "candid", - "ic-agent", + "ic-agent 0.37.1", "k256", "rate-limits-api", "regex", @@ -1929,7 +1929,7 @@ dependencies = [ "flate2", "futures", "http 1.1.0", - "ic-agent", + "ic-agent 0.37.1", "ic-http-certification", "ic-response-verification", "ic-utils 0.37.0", @@ -2354,8 +2354,8 @@ dependencies = [ "anyhow", "clap 4.5.20", "config_types", - "deterministic_ips", "ic-types", + "macaddr", "network", "once_cell", "regex", @@ -2372,13 +2372,14 @@ name = "config_types" version = "1.0.0" dependencies = [ "anyhow", - "deterministic_ips", "ic-types", + "macaddr", "once_cell", "serde", "serde_json", "serde_with 1.14.0", "tempfile", + "thiserror 2.0.3", "url", ] @@ -2409,7 +2410,7 @@ dependencies = [ "anyhow", "canister-test", "canister_http", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-interfaces-registry", "ic-nns-common", @@ -2441,7 +2442,7 @@ dependencies = [ "candid", "canister-test", "canister_http", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-canister-client", "ic-management-canister-types", @@ -2469,7 +2470,7 @@ dependencies = [ "canister-test", "chrono", "futures", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-config", "ic-limits", @@ -2502,7 +2503,7 @@ dependencies = [ "anyhow", "candid", "futures", - "ic-agent", + "ic-agent 0.37.1", "ic-canister-client", "ic-management-canister-types", "ic-nervous-system-common-test-keys", @@ -3374,7 +3375,9 @@ version = "0.1.0" dependencies = [ "anyhow", "clap 4.5.20", + "config_types", "ic-crypto-sha2", + "macaddr", "thiserror 2.0.3", ] @@ -4088,7 +4091,7 @@ dependencies = [ "bitcoincore-rpc", "candid", "futures", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-cdk 0.16.0", "ic-config", @@ -4932,6 +4935,7 @@ dependencies = [ "anyhow", "clap 4.5.20", "config", + "config_types", "deterministic_ips", "network", "utils", @@ -5389,7 +5393,7 @@ dependencies = [ "hyper-rustls 0.27.3", "hyper-util", "ic-certification 2.6.0", - "ic-transport-types", + "ic-transport-types 0.37.1", "ic-verify-bls-signature 0.5.0", "k256", "leb128", @@ -5415,6 +5419,50 @@ dependencies = [ "url", ] +[[package]] +name = "ic-agent" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "158138fcb769fe6288e63d5db221c904e472cfb7d376aba13a38c060f2984e63" +dependencies = [ + "async-lock", + "async-trait", + "backoff", + "cached 0.52.0", + "candid", + "der", + "ecdsa", + "ed25519-consensus", + "elliptic-curve", + "futures-util", + "hex", + "http 1.1.0", + "http-body 1.0.1", + "ic-certification 2.6.0", + "ic-transport-types 0.39.1", + "ic-verify-bls-signature 0.5.0", + "k256", + "leb128", + "p256", + "pem 3.0.4", + "pkcs8", + "rand 0.8.5", + "rangemap", + "reqwest 0.12.9", + "sec1", + "serde", + "serde_bytes", + "serde_cbor", + "serde_repr", + "sha2 0.10.8", + "simple_asn1", + "thiserror 1.0.68", + "time", + "tokio", + "tower-service", + "url", +] + [[package]] name = "ic-artifact-downloader" version = "0.1.0" @@ -5616,7 +5664,7 @@ dependencies = [ [[package]] name = "ic-bn-lib" version = "0.1.0" -source = "git+https://github.com/dfinity/ic-bn-lib?rev=2496803db52d8182f578f098a3ffe4fcb9573fcd#2496803db52d8182f578f098a3ffe4fcb9573fcd" +source = "git+https://github.com/dfinity/ic-bn-lib?rev=526d34d15cfbf369d8baf2dae9932aa18d570a1d#526d34d15cfbf369d8baf2dae9932aa18d570a1d" dependencies = [ "ahash 0.8.11", "anyhow", @@ -5664,7 +5712,7 @@ dependencies = [ "strum_macros", "sync_wrapper 1.0.1", "systemstat", - "thiserror 2.0.3", + "thiserror 1.0.68", "tokio", "tokio-io-timeout", "tokio-rustls 0.26.0", @@ -5684,6 +5732,7 @@ dependencies = [ name = "ic-boundary" version = "0.9.0" dependencies = [ + "anonymization-client", "anyhow", "arc-swap", "async-scoped", @@ -5702,14 +5751,20 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "humantime", + "ic-agent 0.39.1", "ic-base-types", "ic-bn-lib", + "ic-canister-client", + "ic-canister-client-sender", "ic-certification-test-utils", "ic-config", + "ic-crypto", "ic-crypto-ed25519", "ic-crypto-tree-hash", + "ic-crypto-utils-basic-sig", "ic-crypto-utils-threshold-sig-der", "ic-crypto-utils-tls", + "ic-interfaces", "ic-interfaces-registry", "ic-limits", "ic-logger", @@ -5773,7 +5828,7 @@ dependencies = [ "candid", "certificate_orchestrator_interface", "chacha20poly1305", - "ic-agent", + "ic-agent 0.37.1", "ic-interfaces-registry", "ic-protobuf", "ic-registry-keys", @@ -5798,7 +5853,7 @@ version = "0.9.0" dependencies = [ "anyhow", "futures", - "ic-agent", + "ic-agent 0.37.1", "ic-boundary-nodes-system-test-utils", "ic-crypto-tree-hash", "ic-system-test-driver", @@ -5817,7 +5872,7 @@ version = "0.9.0" dependencies = [ "anyhow", "candid", - "ic-agent", + "ic-agent 0.37.1", "ic-boundary-nodes-system-test-utils", "ic-protobuf", "ic-registry-keys", @@ -5838,7 +5893,7 @@ version = "0.9.0" dependencies = [ "anyhow", "futures", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-interfaces-registry", "ic-protobuf", @@ -5990,12 +6045,15 @@ dependencies = [ "hex", "ic-base-types", "ic-btc-interface", + "ic-canister-log 0.2.0", "ic-canisters-http-types", "ic-cdk 0.16.0", + "ic-metrics-encoder", "ic-stable-structures", "ic-test-utilities-load-wasm", "ic-types", "ic-universal-canister", + "num-traits", "pocket-ic", "scraper", "serde", @@ -6065,6 +6123,7 @@ dependencies = [ "ic-crypto-tree-hash", "ic-management-canister-types", "ic-protobuf", + "ic-read-state-response-parser", "ic-test-utilities", "ic-test-utilities-types", "ic-types", @@ -6522,7 +6581,7 @@ name = "ic-ckbtc-agent" version = "0.9.0" dependencies = [ "candid", - "ic-agent", + "ic-agent 0.37.1", "ic-canisters-http-types", "ic-ckbtc-minter", "ic-icrc1", @@ -6570,7 +6629,7 @@ dependencies = [ "ciborium", "flate2", "hex", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-bitcoin-canister-mock", "ic-btc-interface", @@ -6627,7 +6686,7 @@ dependencies = [ "futures", "hex", "hex-literal", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-canister-log 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "ic-canisters-http-types", @@ -8465,6 +8524,7 @@ dependencies = [ "ic-metrics", "ic-pprof", "ic-protobuf", + "ic-read-state-response-parser", "ic-registry-client-helpers", "ic-registry-keys", "ic-registry-provisional-whitelist", @@ -8556,7 +8616,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "http-body-util", - "ic-agent", + "ic-agent 0.37.1", "ic-http-certification", "ic-response-verification", "ic-utils 0.37.0", @@ -8781,7 +8841,7 @@ dependencies = [ "clap 4.5.20", "futures", "hex", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-icrc-rosetta-client", "ic-icrc-rosetta-runner", @@ -8837,7 +8897,7 @@ dependencies = [ "candid", "clap 4.5.20", "hex", - "ic-agent", + "ic-agent 0.37.1", "ic-crypto-ed25519", "ic-crypto-secp256k1", "ic-icrc-rosetta", @@ -9021,7 +9081,7 @@ dependencies = [ "candid", "candid_parser", "ciborium", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-canister-log 0.2.0", "ic-canister-profiler", @@ -9067,7 +9127,7 @@ dependencies = [ "cddl", "ciborium", "hex", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-canister-log 0.2.0", "ic-canisters-http-types", @@ -9103,7 +9163,7 @@ name = "ic-icrc1-test-utils" version = "0.9.0" dependencies = [ "candid", - "ic-agent", + "ic-agent 0.37.1", "ic-crypto-ed25519", "ic-crypto-secp256k1", "ic-crypto-test-utils-reproducible-rng", @@ -9315,7 +9375,7 @@ dependencies = [ "chrono", "ciborium", "dfn_protobuf", - "ic-agent", + "ic-agent 0.37.1", "ic-certification 0.9.0", "ic-crypto-sha2", "ic-ledger-canister-blocks-synchronizer-test-utils", @@ -9461,7 +9521,7 @@ dependencies = [ "cddl", "futures", "hex", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-canisters-http-types", "ic-config", @@ -9728,7 +9788,7 @@ version = "0.0.1" dependencies = [ "anyhow", "candid", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-nervous-system-clients", "ic-nns-common", @@ -10107,7 +10167,7 @@ name = "ic-networking-subnet-update-workload" version = "0.9.0" dependencies = [ "anyhow", - "ic-agent", + "ic-agent 0.37.1", "ic-interfaces-registry", "ic-protobuf", "ic-registry-canister-api", @@ -10146,7 +10206,7 @@ version = "0.9.0" dependencies = [ "candid", "colored", - "ic-agent", + "ic-agent 0.37.1", "ic-neurons-fund", "ic-sns-governance", "ic-sns-swap", @@ -10582,11 +10642,13 @@ dependencies = [ "ic-xrc-types", "icp-ledger", "icrc-ledger-types", + "itertools 0.12.1", "lazy_static", "lifeline", "maplit", "on_wire", "phantom_newtype", + "pocket-ic", "prometheus-parse", "prost 0.13.3", "rand 0.8.5", @@ -10907,6 +10969,20 @@ dependencies = [ "turmoil", ] +[[package]] +name = "ic-read-state-response-parser" +version = "0.9.0" +dependencies = [ + "ic-canonical-state", + "ic-certification 0.9.0", + "ic-certification-test-utils", + "ic-crypto-tree-hash", + "ic-types", + "serde", + "serde_cbor", + "tree-deserializer", +] + [[package]] name = "ic-recovery" version = "0.1.0" @@ -11585,9 +11661,8 @@ dependencies = [ "dfn_protobuf", "futures", "hex", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", - "ic-canister-client", "ic-crypto-ed25519", "ic-crypto-sha2", "ic-crypto-tree-hash", @@ -11611,6 +11686,7 @@ dependencies = [ "ic-nns-governance-init", "ic-nns-handler-root", "ic-nns-test-utils", + "ic-read-state-response-parser", "ic-rosetta-test-utils", "ic-types", "icp-ledger", @@ -11727,7 +11803,7 @@ dependencies = [ "candid", "colored", "csv", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-nervous-system-agent", "ic-nervous-system-common-test-keys", @@ -11755,7 +11831,7 @@ dependencies = [ "clap 4.5.20", "futures", "hex", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-crypto-sha2", "ic-nervous-system-agent", @@ -11856,6 +11932,7 @@ dependencies = [ "strum", "strum_macros", "tempfile", + "time", "tokio", "tokio-test", ] @@ -12531,7 +12608,7 @@ version = "0.1.0" dependencies = [ "clap 4.5.20", "hex", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-crypto-utils-threshold-sig", "ic-crypto-utils-threshold-sig-der", @@ -12626,6 +12703,7 @@ dependencies = [ "canister-test", "chrono", "clap 4.5.20", + "config_types", "crossbeam-channel", "cycles-minting-canister", "deterministic_ips", @@ -12638,7 +12716,7 @@ dependencies = [ "humantime", "humantime-serde", "hyper 1.5.1", - "ic-agent", + "ic-agent 0.37.1", "ic-artifact-pool", "ic-base-types", "ic-btc-interface", @@ -12714,6 +12792,7 @@ dependencies = [ "lazy_static", "leb128", "lifeline", + "macaddr", "maplit", "nix 0.24.3", "num_cpus", @@ -13069,7 +13148,7 @@ dependencies = [ "candid", "canister-test", "dfn_candid", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-btc-interface", "ic-btc-kyt", @@ -13155,6 +13234,24 @@ dependencies = [ "thiserror 1.0.68", ] +[[package]] +name = "ic-transport-types" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d8789a5c176bb1b925fa58ca97c651a3995d504e76101e93d2a17f558bdcf66" +dependencies = [ + "candid", + "hex", + "ic-certification 2.6.0", + "leb128", + "serde", + "serde_bytes", + "serde_cbor", + "serde_repr", + "sha2 0.10.8", + "thiserror 1.0.68", +] + [[package]] name = "ic-tvl-canister" version = "0.9.0" @@ -13270,7 +13367,7 @@ dependencies = [ "async-trait", "candid", "futures-util", - "ic-agent", + "ic-agent 0.37.1", "once_cell", "semver", "serde", @@ -13668,7 +13765,7 @@ dependencies = [ "anyhow", "candid", "certificate_orchestrator_interface", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-boundary-nodes-integration-test-common", "ic-boundary-nodes-performance-test-common", @@ -13712,7 +13809,7 @@ version = "0.9.0" dependencies = [ "anyhow", "futures", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-system-test-driver", "ic-types", @@ -13731,7 +13828,7 @@ dependencies = [ "canister-test", "chrono", "futures", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-canister-client", "ic-config", @@ -13771,7 +13868,7 @@ dependencies = [ "candid", "canister-test", "futures", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-crypto-test-utils-reproducible-rng", "ic-management-canister-types", @@ -13809,7 +13906,7 @@ dependencies = [ "candid", "canister-test", "ed25519-dalek", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-canister-client", "ic-config", @@ -13840,7 +13937,7 @@ version = "0.9.0" dependencies = [ "anyhow", "candid", - "ic-agent", + "ic-agent 0.37.1", "ic-crypto-test-utils-reproducible-rng", "ic-fstrim-tool", "ic-registry-subnet-type", @@ -13937,7 +14034,7 @@ version = "0.9.0" dependencies = [ "anyhow", "candid", - "ic-agent", + "ic-agent 0.37.1", "ic-icrc-rosetta", "ic-icrc-rosetta-client", "ic-ledger-test-utils", @@ -13975,7 +14072,7 @@ dependencies = [ "candid", "ciborium", "hex", - "ic-agent", + "ic-agent 0.37.1", "ic-cbor", "ic-certification 2.6.0", "icrc-ledger-types", @@ -14891,7 +14988,7 @@ dependencies = [ "dfn_http_metrics", "dfn_protobuf", "hex", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-canister-log 0.2.0", "ic-cdk 0.16.0", @@ -15330,6 +15427,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "macaddr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baee0bbc17ce759db233beb01648088061bf678383130602a298e6998eedb2d8" + [[package]] name = "mach2" version = "0.4.2" @@ -15510,8 +15613,9 @@ dependencies = [ "candid", "canister-test", "dfn_candid", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", + "ic-cdk 0.16.0", "ic-registry-subnet-type", "ic-system-test-driver", "ic-types", @@ -15777,6 +15881,7 @@ dependencies = [ "anyhow", "config_types", "deterministic_ips", + "macaddr", "ping", "rayon", "regex", @@ -15797,7 +15902,7 @@ dependencies = [ "canister_http", "cloner-canister-types", "dfn_candid", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-cdk 0.16.0", "ic-limits", @@ -15913,7 +16018,7 @@ dependencies = [ "cycles_minting", "dfn_candid", "futures", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-btc-interface", "ic-canister-client", @@ -15986,7 +16091,7 @@ version = "0.9.0" dependencies = [ "anyhow", "candid", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-nns-constants", "ic-registry-canister-api", @@ -16525,6 +16630,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_cbor", + "serde_json", "slog", "slog-async", "strum", @@ -17091,7 +17197,7 @@ dependencies = [ "ic-cdk 0.16.0", "ic-certification 2.6.0", "ic-error-types", - "ic-transport-types", + "ic-transport-types 0.37.1", "k256", "lazy_static", "reqwest 0.12.9", @@ -17140,7 +17246,7 @@ dependencies = [ "http-body-util", "hyper 1.5.1", "hyper-util", - "ic-agent", + "ic-agent 0.37.1", "ic-boundary", "ic-btc-adapter", "ic-btc-interface", @@ -18637,7 +18743,7 @@ dependencies = [ "anyhow", "candid", "hex", - "ic-agent", + "ic-agent 0.37.1", "ic-crypto-ed25519", "ic-crypto-secp256k1", "ic-types", @@ -19574,6 +19680,7 @@ dependencies = [ "anyhow", "clap 4.5.20", "config", + "config_types", "deterministic_ips", "network", "utils", @@ -19873,7 +19980,7 @@ version = "0.9.0" dependencies = [ "anyhow", "candid", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-canister-client", "ic-canister-client-sender", @@ -20263,7 +20370,7 @@ dependencies = [ "candid", "colored", "futures", - "ic-agent", + "ic-agent 0.37.1", "ic-base-types", "ic-nervous-system-agent", "ic-nervous-system-clients", @@ -20358,9 +20465,11 @@ dependencies = [ name = "systest-message-routing-common" version = "0.9.0" dependencies = [ + "candid", "canister-test", "dfn_candid", "futures", + "ic-cdk 0.16.0", "ic-system-test-driver", "slog", "xnet-test", @@ -20590,7 +20699,7 @@ dependencies = [ "futures", "hex", "humantime", - "ic-agent", + "ic-agent 0.37.1", "ic-async-utils", "ic-base-types", "ic-btc-interface", @@ -22807,7 +22916,9 @@ name = "xnet-test" version = "0.9.0" dependencies = [ "candid", - "dfn_core", + "futures", + "ic-cdk 0.16.0", + "ic-cdk-macros 0.9.0", "rand 0.8.5", "rand_pcg 0.3.1", "serde", diff --git a/Cargo.toml b/Cargo.toml index 9bd04ae4d02..5f9fe61f536 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ members = [ "rs/boundary_node/rate_limits/canister_client", "rs/boundary_node/systemd_journal_gatewayd_shim", "rs/canister_client", + "rs/canister_client/read_state_response_parser", "rs/canister_client/sender", "rs/cross-chain/proposal-cli", "rs/cycles_account_manager", @@ -606,6 +607,7 @@ libc = "0.2.158" libflate = "2.1.0" libnss = "0.5.0" local-ip-address = "0.5.6" +macaddr = "1.0" minicbor = { version = "0.19.1", features = ["alloc", "derive"] } minicbor-derive = "0.13.0" mockall = "0.13.0" diff --git a/MODULE.bazel b/MODULE.bazel index dc0907146ef..1aee2d6536a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -37,6 +37,10 @@ archive_override( urls = ["https://github.com/bazelbuild/rules_foreign_cc/archive/77d4483fadbb1b7bcace18ed8e8e87e8791050f6.tar.gz"], ) +# Misc tools + +bazel_dep(name = "pigz", version = "2.8") # (parallel) gzip + # Python dependencies bazel_dep(name = "rules_python", version = "0.35.0") @@ -429,38 +433,6 @@ http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_ http_jar = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") -# Gzip tool -http_archive( - name = "pigz", - build_file_content = """ -cc_library( - name = "zopfli", - hdrs = glob(["zopfli/src/zopfli/*.h"]), - srcs = glob(["zopfli/src/zopfli/*.c"]), -) - -cc_binary( - name = "pigz", - srcs = [ - "pigz.c", - "try.c", - "try.h", - "yarn.c", - "yarn.h", - ], - linkopts = [ - "-lm", - "-lpthread", - "-lz", - ], - deps = [":zopfli"], - visibility = ["//visibility:public"], -)""", - sha256 = "688fe1d805b33a4ae8cbf86ba56aa1a5d647e959219c7d3aeb7d041c228af1ef", - strip_prefix = "pigz-2.8", - url = "https://github.com/madler/pigz/archive/refs/tags/v2.8.zip", -) - # TLA+ tools http_jar( name = "tlaplus_community_modules", diff --git a/bazel/external_crates.bzl b/bazel/external_crates.bzl index a754f976caf..bb628cdcc0b 100644 --- a/bazel/external_crates.bzl +++ b/bazel/external_crates.bzl @@ -52,15 +52,18 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable ], )], "lmdb-rkv-sys": [crate.annotation( + # patch our fork of the lmdb-rkv-sys to allow specifying the path + # to the built static archive + patch_args = ["-p1"], + patches = ["@@//bazel:lmdb_rkv_sys.patch"], build_script_data = [ + "@lmdb//:liblmdb", "@lmdb//:lmdb.h", ], build_script_env = { - "LMDB_NO_BUILD": "1", + "LMDB_OVERRIDE": "$(location @lmdb//:liblmdb)", "LMDB_H_PATH": "$(location @lmdb//:lmdb.h)", }, - # ensure LMDB lib is available at runtime - deps = ["@lmdb"], )], "p256": [crate.annotation( rustc_flags = [ @@ -764,6 +767,9 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable version = "^0.7.8", default_features = False, ), + "macaddr": crate.spec( + version = "^1.0", + ), "mach2": crate.spec( # Wasmtime depends on 0.4.2 but specifies 0.4.1. # Enforce 0.4.2 using a dummy dependency until diff --git a/bazel/lmdb_rkv_sys.patch b/bazel/lmdb_rkv_sys.patch new file mode 100644 index 00000000000..c55f134c782 --- /dev/null +++ b/bazel/lmdb_rkv_sys.patch @@ -0,0 +1,29 @@ +# patch our fork of the lmdb-rkv-sys to allow specifying the path +# to the built static archive +diff --git a/build.rs b/build.rs +index c422d52..b779ee0 100644 +--- a/build.rs ++++ b/build.rs +@@ -57,7 +57,21 @@ fn main() { + warn!("Building with `-fsanitize=fuzzer`."); + } + +- if let Err(_) = std::env::var("LMDB_NO_BUILD") { ++ if let Ok(lmdb) = std::env::var("LMDB_OVERRIDE") { ++ let lmdb = PathBuf::from(lmdb); ++ assert!( ++ lmdb.exists(), ++ "Path to `lmdb` '{}' does not exist", ++ lmdb.display() ++ ); ++ println!( ++ "cargo:rustc-link-search=native={}", ++ lmdb.parent().unwrap().display() ++ ); ++ let stem = lmdb.file_stem().unwrap().to_str().unwrap(); ++ println!("cargo:rustc-link-lib=static={}", &stem[3..]); ++ return; ++ } else { + if pkg_config::probe_library("lmdb").is_err() { + let mut lmdb = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); + lmdb.push("lmdb"); diff --git a/bazel/upload_systest_dep.sh b/bazel/upload_systest_dep.sh index 68f092947f3..a1d9b6788b4 100755 --- a/bazel/upload_systest_dep.sh +++ b/bazel/upload_systest_dep.sh @@ -64,7 +64,7 @@ else dep_upload_url="$UPLOAD_URL/$dep_sha256" echo "Using upload URL: '$dep_upload_url'" >&2 curl_out=$(mktemp) - curl --silent --fail "$dep_upload_url" --upload-file "$dep_filename" -w '%{size_upload} %{time_total} %{speed_upload}\n' | tee "$curl_out" >&2 + curl --silent --show-error --fail --retry 3 "$dep_upload_url" --upload-file "$dep_filename" -w '%{size_upload} %{time_total} %{speed_upload}\n' | tee "$curl_out" >&2 # read & pretty print 3 metrics: upload size, upload time & upload speed if read -ra metrics <"$curl_out"; then echo "Uploaded $(numfmt --to=iec-i --suffix=B "${metrics[0]}") in ${metrics[1]}s ($(numfmt --to=iec-i --suffix=B "${metrics[2]}")/s)" >&2 diff --git a/ci/bazel-scripts/main.sh b/ci/bazel-scripts/main.sh index 74fc662cd68..4da4bafd554 100755 --- a/ci/bazel-scripts/main.sh +++ b/ci/bazel-scripts/main.sh @@ -30,7 +30,7 @@ if [[ "${IS_PROTECTED_BRANCH:-}" == "true" ]] || [[ "${CI_PULL_REQUEST_TARGET_BR RUN_ON_DIFF_ONLY="false" fi -if [[ "${CI_PIPELINE_SOURCE:-}" == "merge_group" ]]; then +if [[ "${CI_EVENT_NAME:-}" == "merge_group" ]]; then s3_upload="False" RUN_ON_DIFF_ONLY="false" fi diff --git a/ci/container/Dockerfile b/ci/container/Dockerfile index 8e9293254b3..d555d8f6694 100644 --- a/ci/container/Dockerfile +++ b/ci/container/Dockerfile @@ -185,7 +185,7 @@ RUN curl --fail https://sh.rustup.rs -sSf \ RUN rm -rf /tmp/rust-version # Add cargo-audit -ARG CARGO_AUDIT_VERSION=0.20.0 +ARG CARGO_AUDIT_VERSION=0.21.0 RUN cargo install cargo-audit --version ${CARGO_AUDIT_VERSION} # Add zshrc generated from zsh-newuser-install (option 2) COPY --chown=ubuntu:ubuntu ./ci/container/files/zshrc /home/ubuntu/.zshrc diff --git a/ci/container/TAG b/ci/container/TAG index 68b377acf0e..6e197c4645e 100644 --- a/ci/container/TAG +++ b/ci/container/TAG @@ -1 +1 @@ -f6b02fd6aa4c143cc46fe7d66403d704903a18f5f05cf987d9f33c04cbc732c7 +234270bdfedab79b1aa7a761cad77195e52c801b287b6c3cdf2beb3239e8cc14 diff --git a/ci/scripts/lock-generate.sh b/ci/scripts/lock-generate.sh index 7cd36e2ee81..915a002e37b 100755 --- a/ci/scripts/lock-generate.sh +++ b/ci/scripts/lock-generate.sh @@ -16,7 +16,7 @@ git status if ! git diff --cached --quiet; then # If this is running from a pull request then update the Cargo.lock file in the PR # automatically. - if [ "$CI_PIPELINE_SOURCE" = "pull_request" ]; then + if [ "$CI_EVENT_NAME" = "pull_request" ]; then # There are some changes staged git config --global user.email "infra+github-automation@dfinity.org" git config --global user.name "IDX GitHub Automation" diff --git a/ci/scripts/rust-lint.sh b/ci/scripts/rust-lint.sh index 8eb35349496..362c93ad041 100755 --- a/ci/scripts/rust-lint.sh +++ b/ci/scripts/rust-lint.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -xeuo pipefail -cd "$CI_PROJECT_DIR" +cd "${CI_PROJECT_DIR:-$(git rev-parse --show-toplevel)}" cargo fmt -- --check cargo clippy --locked --all-features --workspace --all-targets -- \ -D warnings \ diff --git a/ic-os/boundary-guestos/context/docker-base.prod b/ic-os/boundary-guestos/context/docker-base.prod index a7ab66574b3..b9518202a1c 100644 --- a/ic-os/boundary-guestos/context/docker-base.prod +++ b/ic-os/boundary-guestos/context/docker-base.prod @@ -1 +1 @@ -ghcr.io/dfinity/boundaryos-base@sha256:d2034fde545b18002e08ed7f80af8beec12ca682f76f5a2fe1c3d41c6a32cbf4 +ghcr.io/dfinity/boundaryos-base@sha256:d97d5a2e72fb0c80643a707708299ae392f54ef5b41e9d0292129d9e47d52a93 diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 5dd87e9a334..024fc730284 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -84,8 +84,7 @@ function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) cmd+=(--nns_public_key "/boot/config/nns_public_key.pem") cmd+=(--elasticsearch_hosts "$(/opt/ic/bin/fetch-property.sh --key=.logging.hosts --metric=hostos_logging_hosts --config=${DEPLOYMENT})") - # 1 corresponds to GuestOS - cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type 1)") + cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") @@ -110,8 +109,7 @@ function assemble_config_media() { function generate_guestos_config() { RESOURCES_MEMORY=$(/opt/ic/bin/fetch-property.sh --key=.resources.memory --metric=hostos_resources_memory --config=${DEPLOYMENT}) - # 1 corresponds to GuestOS - MAC_ADDRESS=$(/opt/ic/bin/hostos_tool generate-mac-address --node-type 1) + MAC_ADDRESS=$(/opt/ic/bin/hostos_tool generate-mac-address --node-type GuestOS) # NOTE: `fetch-property` will error if the target is not found. Here we # only want to act when the field is set. CPU_MODE=$(jq -r ".resources.cpu" ${DEPLOYMENT}) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 883beb5116e..3d125938234 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -84,8 +84,7 @@ function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) cmd+=(--nns_public_key "/boot/config/nns_public_key.pem") cmd+=(--elasticsearch_hosts "$(/opt/ic/bin/fetch-property.sh --key=.logging.hosts --metric=hostos_logging_hosts --config=${DEPLOYMENT})") - # 1 corresponds to GuestOS - cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type 1)") + cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") @@ -108,8 +107,7 @@ function assemble_config_media() { function generate_guestos_config() { RESOURCES_MEMORY=$(/opt/ic/bin/fetch-property.sh --key=.resources.memory --metric=hostos_resources_memory --config=${DEPLOYMENT}) - # 1 corresponds to GuestOS - MAC_ADDRESS=$(/opt/ic/bin/hostos_tool generate-mac-address --node-type 1) + MAC_ADDRESS=$(/opt/ic/bin/hostos_tool generate-mac-address --node-type GuestOS) # NOTE: `fetch-property` will error if the target is not found. Here we # only want to act when the field is set. CPU_MODE=$(jq -r ".resources.cpu" ${DEPLOYMENT}) diff --git a/ic-os/components/ic/share/ic-boundary.env b/ic-os/components/ic/share/ic-boundary.env index d82b644f66e..570be9bb8ae 100644 --- a/ic-os/components/ic/share/ic-boundary.env +++ b/ic-os/components/ic/share/ic-boundary.env @@ -1,10 +1,11 @@ -LISTEN_HTTP_PORT="443" +LISTEN_HTTPS_PORT="443" TLS_CERT_PATH="/var/lib/ic/data/ic-boundary-tls.crt" TLS_PKEY_PATH="/var/lib/ic/data/ic-boundary-tls.key" TLS_ACME_CREDENTIALS_PATH="/var/lib/ic/data" HTTP_CLIENT_TIMEOUT_CONNECT="3s" REGISTRY_DISABLE_REPLICATOR="true" REGISTRY_LOCAL_STORE_PATH="/var/lib/ic/data/ic_registry_local_store" +OBS_LOG_ANONYMIZATION_CANISTER_ID="uz2z3-qyaaa-aaaaq-qaacq-cai" OBS_LOG_JOURNALD="true" OBS_METRICS_ADDR="[::]:9324" RATE_LIMIT_PER_SECOND_PER_SUBNET="1000" diff --git a/ic-os/components/setupos-scripts/check-network.sh b/ic-os/components/setupos-scripts/check-network.sh index aeb8c80fc6f..094cb5e6db4 100755 --- a/ic-os/components/setupos-scripts/check-network.sh +++ b/ic-os/components/setupos-scripts/check-network.sh @@ -101,9 +101,8 @@ function get_network_settings() { "echo ${ipv6_address_system_full} | awk -F '/' '{ print \$1 }'" \ "Failed to get system's IPv6 address.") - # 0 corresponds to HostOS, 1 to GuestOS - HOSTOS_IPV6_ADDRESS=$(/opt/ic/bin/setupos_tool generate-ipv6-address --node-type 0) - GUESTOS_IPV6_ADDRESS=$(/opt/ic/bin/setupos_tool generate-ipv6-address --node-type 1) + HOSTOS_IPV6_ADDRESS=$(/opt/ic/bin/setupos_tool generate-ipv6-address --node-type HostOS) + GUESTOS_IPV6_ADDRESS=$(/opt/ic/bin/setupos_tool generate-ipv6-address --node-type GuestOS) } function print_network_settings() { diff --git a/ic-os/components/upgrade/shared-resources/upgrade-shared-data-store/upgrade-shared-data-store.service b/ic-os/components/upgrade/shared-resources/upgrade-shared-data-store/upgrade-shared-data-store.service index 99c243c2601..f516ebe3217 100644 --- a/ic-os/components/upgrade/shared-resources/upgrade-shared-data-store/upgrade-shared-data-store.service +++ b/ic-os/components/upgrade/shared-resources/upgrade-shared-data-store/upgrade-shared-data-store.service @@ -3,6 +3,7 @@ Description=Initialize node data storage DefaultDependencies=no Requires=var-lib-ic-data.mount After=var-lib-ic-data.mount +Before=setup-permissions.service [Install] WantedBy=local-fs.target diff --git a/ic-os/components/upgrade/shared-resources/upgrade-shared-data-store/upgrade-shared-data-store.sh b/ic-os/components/upgrade/shared-resources/upgrade-shared-data-store/upgrade-shared-data-store.sh index 45465d845b7..80f25ae5ed9 100755 --- a/ic-os/components/upgrade/shared-resources/upgrade-shared-data-store/upgrade-shared-data-store.sh +++ b/ic-os/components/upgrade/shared-resources/upgrade-shared-data-store/upgrade-shared-data-store.sh @@ -8,6 +8,6 @@ set -e # Fix up ownership -- should be owned by the ic replica service user. USER=$(stat -c %U /var/lib/ic/data) -if [ "${USER}" != replica ]; then +if [ "${USER}" != ic-replica ]; then chown -R ic-replica /var/lib/ic/data fi diff --git a/ic-os/guestos/context/docker-base.dev b/ic-os/guestos/context/docker-base.dev index 55a9f389b70..d656fa23660 100644 --- a/ic-os/guestos/context/docker-base.dev +++ b/ic-os/guestos/context/docker-base.dev @@ -1 +1 @@ -ghcr.io/dfinity/guestos-base-dev@sha256:414eb3ecbe890440815901f36fd3bc371e44551d6a373b4d0b66e6fc22dd056d +ghcr.io/dfinity/guestos-base-dev@sha256:981c653650c242f5490fd6c9ea46525438e472dc44c750963eac02fca321c19e diff --git a/ic-os/guestos/context/docker-base.prod b/ic-os/guestos/context/docker-base.prod index c7a2cde9caf..4dac2a92683 100644 --- a/ic-os/guestos/context/docker-base.prod +++ b/ic-os/guestos/context/docker-base.prod @@ -1 +1 @@ -ghcr.io/dfinity/guestos-base@sha256:f005a27883afc2c265d69b4312702a95d719a0d6bbcbd52840b547e6fba7bd1f +ghcr.io/dfinity/guestos-base@sha256:f932d313bb455a59b79f78b56bbb96cd97a12e3903dd96ee8a05b377633680c7 diff --git a/ic-os/hostos/context/docker-base.dev b/ic-os/hostos/context/docker-base.dev index c12827229c1..3e47b0c8f9d 100644 --- a/ic-os/hostos/context/docker-base.dev +++ b/ic-os/hostos/context/docker-base.dev @@ -1 +1 @@ -ghcr.io/dfinity/hostos-base-dev@sha256:d7308919c23c5936fb6570db23658055cc02e0620d30c1b695984062ef9fe09e +ghcr.io/dfinity/hostos-base-dev@sha256:5cf9998a41ca100fe8b5c5e0678aa1230e1b8cb8e316097c8f1ecad1ceafe665 diff --git a/ic-os/hostos/context/docker-base.prod b/ic-os/hostos/context/docker-base.prod index b5babe65219..b4ca677d97f 100644 --- a/ic-os/hostos/context/docker-base.prod +++ b/ic-os/hostos/context/docker-base.prod @@ -1 +1 @@ -ghcr.io/dfinity/hostos-base@sha256:e8eff4fc294495c75cf42053579f6c3e78ca8c682f889ffbdbc1cd917384efff +ghcr.io/dfinity/hostos-base@sha256:5e8b448ab1c2e691e3d9c9bb30ebbaf7e94c095ec9932e0bcacddbaccfd4fe35 diff --git a/ic-os/setupos/context/docker-base.dev b/ic-os/setupos/context/docker-base.dev index f21fab8cefc..18c05f5be1a 100644 --- a/ic-os/setupos/context/docker-base.dev +++ b/ic-os/setupos/context/docker-base.dev @@ -1 +1 @@ -ghcr.io/dfinity/setupos-base-dev@sha256:d5e49439fecbdf0b34ce683de2b7a891a418d629f05fe7a9369c7e485a5e3503 +ghcr.io/dfinity/setupos-base-dev@sha256:fa624ad95f31d22a3e81f66d01eeb47ec43d5583a813103b6cc0d00e2d4838f8 diff --git a/ic-os/setupos/context/docker-base.prod b/ic-os/setupos/context/docker-base.prod index 1fcc65b5129..45bea5e2cea 100644 --- a/ic-os/setupos/context/docker-base.prod +++ b/ic-os/setupos/context/docker-base.prod @@ -1 +1 @@ -ghcr.io/dfinity/setupos-base@sha256:2f507b6331804e00a4fcbaee56c3450a6e62c5f4f6fa5a62a2d7d02c2fd9527c +ghcr.io/dfinity/setupos-base@sha256:191b77c1f3cd293eb0eb255382866e15c1ef7f19419e884166b7e55a2bb50a2d diff --git a/packages/pocket-ic/src/lib.rs b/packages/pocket-ic/src/lib.rs index a2144c0b838..e647de89523 100644 --- a/packages/pocket-ic/src/lib.rs +++ b/packages/pocket-ic/src/lib.rs @@ -140,6 +140,7 @@ impl PocketIcBuilder { .await } + /// Use an already running PocketIC server. pub fn with_server_url(mut self, server_url: Url) -> Self { self.server_url = Some(server_url); self diff --git a/rs/bitcoin/ckbtc/minter/src/lib.rs b/rs/bitcoin/ckbtc/minter/src/lib.rs index 21c55c36232..cd48a462d6f 100644 --- a/rs/bitcoin/ckbtc/minter/src/lib.rs +++ b/rs/bitcoin/ckbtc/minter/src/lib.rs @@ -19,6 +19,7 @@ use num_traits::ToPrimitive; use scopeguard::{guard, ScopeGuard}; use serde::Serialize; use serde_bytes::ByteBuf; +use std::cmp::max; use std::collections::{BTreeMap, BTreeSet}; use std::time::Duration; @@ -52,10 +53,13 @@ pub const MIN_PENDING_REQUESTS: usize = 20; pub const MAX_REQUESTS_PER_BATCH: usize = 100; /// The constants used to compute the minter's fee to cover its own cycle consumption. -/// The values are set to cover the cycle cost on a 28-node subnet. -pub const MINTER_FEE_PER_INPUT: u64 = 246; -pub const MINTER_FEE_PER_OUTPUT: u64 = 7; -pub const MINTER_FEE_CONSTANT: u64 = 52; +pub const MINTER_FEE_PER_INPUT: u64 = 146; +pub const MINTER_FEE_PER_OUTPUT: u64 = 4; +pub const MINTER_FEE_CONSTANT: u64 = 26; +/// Dust limit for the minter's address. +/// The minter's address is of type P2WPKH which means it has a dust limit of 294 sats. +/// For additional safety, we round that value up. +pub const MINTER_ADDRESS_DUST_LIMIT: Satoshi = 300; /// The minimum fee increment for transaction resubmission. /// See https://en.bitcoin.it/wiki/Miner_fees#Relaying for more detail. @@ -998,9 +1002,7 @@ pub fn build_unsigned_transaction( debug_assert!(inputs_value >= amount); - let minter_fee = MINTER_FEE_PER_INPUT * utxos_guard.len() as u64 - + MINTER_FEE_PER_OUTPUT * (outputs.len() + 1) as u64 - + MINTER_FEE_CONSTANT; + let minter_fee = evaluate_minter_fee(utxos_guard.len() as u64, (outputs.len() + 1) as u64); let change = inputs_value - amount; let change_output = state::ChangeOutput { @@ -1076,6 +1078,15 @@ pub fn build_unsigned_transaction( )) } +pub fn evaluate_minter_fee(num_inputs: u64, num_outputs: u64) -> Satoshi { + max( + MINTER_FEE_PER_INPUT * num_inputs + + MINTER_FEE_PER_OUTPUT * num_outputs + + MINTER_FEE_CONSTANT, + MINTER_ADDRESS_DUST_LIMIT, + ) +} + /// Distributes an amount across the specified number of shares as fairly as /// possible. /// @@ -1239,9 +1250,7 @@ pub fn estimate_retrieve_btc_fee( }; let vsize = tx_vsize_estimate(input_count, DEFAULT_OUTPUT_COUNT); - let minter_fee = MINTER_FEE_PER_INPUT * input_count - + MINTER_FEE_PER_OUTPUT * DEFAULT_OUTPUT_COUNT - + MINTER_FEE_CONSTANT; + let minter_fee = evaluate_minter_fee(input_count, DEFAULT_OUTPUT_COUNT); // We subtract one from the outputs because the minter's output // does not participate in fees distribution. let bitcoin_fee = diff --git a/rs/bitcoin/ckbtc/minter/src/tests.rs b/rs/bitcoin/ckbtc/minter/src/tests.rs index 1f306325364..3090e1a1ec3 100644 --- a/rs/bitcoin/ckbtc/minter/src/tests.rs +++ b/rs/bitcoin/ckbtc/minter/src/tests.rs @@ -1,9 +1,9 @@ use crate::state::invariants::CheckInvariantsImpl; -use crate::MINTER_FEE_CONSTANT; use crate::{ address::BitcoinAddress, build_unsigned_transaction, estimate_retrieve_btc_fee, fake_sign, greedy, signature::EncodedSignature, tx, BuildTxError, }; +use crate::{evaluate_minter_fee, MINTER_ADDRESS_DUST_LIMIT}; use crate::{ lifecycle::init::InitArgs, state::{ @@ -17,6 +17,7 @@ use candid::Principal; use ic_base_types::{CanisterId, PrincipalId}; use ic_btc_interface::{Network, OutPoint, Satoshi, Txid, Utxo}; use icrc_ledger_types::icrc1::account::Account; +use maplit::btreeset; use proptest::proptest; use proptest::{ array::uniform20, @@ -258,9 +259,7 @@ fn should_have_same_input_and_output_count() { ) .expect("failed to build a transaction"); - let minter_fee = crate::MINTER_FEE_PER_INPUT * tx.inputs.len() as u64 - + crate::MINTER_FEE_PER_OUTPUT * tx.outputs.len() as u64 - + crate::MINTER_FEE_CONSTANT; + let minter_fee = evaluate_minter_fee(tx.inputs.len() as u64, tx.outputs.len() as u64); assert_eq!(tx.outputs.len(), tx.inputs.len()); assert_eq!( @@ -274,24 +273,22 @@ fn should_have_same_input_and_output_count() { #[test] fn test_min_change_amount() { - let mut available_utxos = BTreeSet::new(); - available_utxos.insert(Utxo { + let utxo_1 = Utxo { outpoint: OutPoint { txid: [0; 32].into(), vout: 0, }, value: 100_000, height: 10, - }); - - available_utxos.insert(Utxo { + }; + let utxo_2 = Utxo { outpoint: OutPoint { txid: [1; 32].into(), vout: 1, }, - value: 100_000, - height: 10, - }); + ..utxo_1.clone() + }; + let mut available_utxos = btreeset! {utxo_1.clone(), utxo_2.clone()}; let minter_addr = BitcoinAddress::P2wpkhV0([0; 20]); let out1_addr = BitcoinAddress::P2wpkhV0([1; 20]); @@ -300,34 +297,43 @@ fn test_min_change_amount() { let (tx, change_output, _) = build_unsigned_transaction( &mut available_utxos, - vec![(out1_addr.clone(), 100_000), (out2_addr.clone(), 99_999)], + vec![ + (out1_addr.clone(), utxo_1.value), + (out2_addr.clone(), utxo_2.value - 1), + ], minter_addr.clone(), fee_per_vbyte, ) .expect("failed to build a transaction"); + let change_value = 1; let fee = fake_sign(&tx).vsize() as u64 * fee_per_vbyte / 1000; - let minter_fee = crate::MINTER_FEE_PER_INPUT * tx.inputs.len() as u64 - + crate::MINTER_FEE_PER_OUTPUT * tx.outputs.len() as u64 - + crate::MINTER_FEE_CONSTANT; + let minter_fee = evaluate_minter_fee(tx.inputs.len() as u64, tx.outputs.len() as u64); assert_eq!(tx.outputs.len(), 3); - let fee_share = (fee + minter_fee - 1) / 2; + let fee_shares = { + let total_fee = fee + minter_fee; + let avg_fee_per_share = total_fee / 2; + let share_1 = avg_fee_per_share + (total_fee % 2); + let share_2 = avg_fee_per_share; + assert_eq!(share_1 + share_2, total_fee); + [share_1, share_2] + }; assert_eq!( &tx.outputs, &[ tx::TxOut { address: out1_addr, - value: 100_000 - fee_share - 1, // Subtract the remainder + value: 100_000 - fee_shares[0], }, tx::TxOut { address: out2_addr, - value: 99_999 - fee_share, + value: 99_999 - fee_shares[1], }, tx::TxOut { address: minter_addr, - value: minter_fee + 1, // Add the remainder + value: minter_fee + change_value, } ] ); @@ -335,57 +341,104 @@ fn test_min_change_amount() { change_output, ChangeOutput { vout: 2, - value: 1 + minter_fee + value: change_value + minter_fee } ); } #[test] fn test_no_dust_outputs() { - let mut available_utxos = BTreeSet::new(); - available_utxos.insert(Utxo { + const P2PKH_DUST_THRESHOLD: u64 = 546; + + let mut available_utxos = btreeset! {Utxo { outpoint: OutPoint { txid: [0; 32].into(), vout: 0, }, value: 100_000, height: 10, - }); + }}; + assert_eq!(available_utxos.len(), 1); + let initial_available_utxos = available_utxos.clone(); let minter_addr = BitcoinAddress::P2wpkhV0([0; 20]); let out1_addr = BitcoinAddress::P2wpkhV0([1; 20]); let out2_addr = BitcoinAddress::P2wpkhV0([2; 20]); - let fee_per_vbyte = 10000; - assert_eq!( - build_unsigned_transaction( - &mut available_utxos, - vec![(out1_addr.clone(), 99_900), (out2_addr.clone(), 100)], - minter_addr.clone(), - fee_per_vbyte, - ), - Err(BuildTxError::DustOutput { - address: out2_addr.clone(), - amount: 100 - }) - ); + for dust in 0..=P2PKH_DUST_THRESHOLD { + let fee_per_vbyte = 10000; + assert_eq!( + build_unsigned_transaction( + &mut available_utxos, + vec![(out1_addr.clone(), 99_000), (out2_addr.clone(), dust)], + minter_addr.clone(), + fee_per_vbyte, + ), + Err(BuildTxError::DustOutput { + address: out2_addr.clone(), + amount: dust + }) + ); + assert_eq!(available_utxos, initial_available_utxos); - let fee_per_vbyte = 4000; + let fee_per_vbyte = 4000; + assert_eq!( + build_unsigned_transaction( + &mut available_utxos, + vec![(out1_addr.clone(), 99_000), (out2_addr.clone(), dust)], + minter_addr.clone(), + fee_per_vbyte, + ), + Err(BuildTxError::DustOutput { + address: out2_addr.clone(), + amount: dust + }) + ); + assert_eq!(available_utxos, initial_available_utxos); + } +} - assert_eq!( - build_unsigned_transaction( +#[test] +fn test_no_dust_in_change_output() { + let utxo = Utxo { + outpoint: OutPoint { + txid: [0; 32].into(), + vout: 0, + }, + value: 100_000, + height: 10, + }; + + let minter_addr = BitcoinAddress::P2wpkhV0([0; 20]); + let out1_addr = BitcoinAddress::P2wpkhV0([1; 20]); + let fee_per_vbyte = 1; + + for change in 1..=100 { + let mut available_utxos = btreeset! {utxo.clone()}; + let (tx, change_output, _utxos) = build_unsigned_transaction( &mut available_utxos, - vec![(out1_addr, 99_000), (out2_addr.clone(), 1000)], - minter_addr, + vec![(out1_addr.clone(), utxo.value - change)], + minter_addr.clone(), fee_per_vbyte, - ), - Err(BuildTxError::DustOutput { - address: out2_addr, - amount: 1000 - }) - ); + ) + .expect("failed to build a transaction"); + let fee = evaluate_minter_fee(tx.inputs.len() as u64, tx.outputs.len() as u64); - assert_eq!(available_utxos.len(), 1); + assert_eq!( + &tx.outputs, + &[ + tx::TxOut { + value: utxo.value - change - fee, + address: out1_addr.clone() + }, + tx::TxOut { + value: change_output.value, + address: minter_addr.clone(), + } + ] + ); + assert!(change_output.value >= change + MINTER_ADDRESS_DUST_LIMIT); + } } fn arb_amount() -> impl Strategy { @@ -663,13 +716,14 @@ proptest! { let target = total_value / 2; + let minter_address= BitcoinAddress::P2wpkhV0(main_pkhash); let fee_estimate = estimate_retrieve_btc_fee(&utxos, Some(target), fee_per_vbyte); let fee_estimate = fee_estimate.minter_fee + fee_estimate.bitcoin_fee; let (unsigned_tx, _, _) = build_unsigned_transaction( &mut utxos, vec![(BitcoinAddress::P2wpkhV0(dst_pkhash), target)], - BitcoinAddress::P2wpkhV0(main_pkhash), + minter_address, fee_per_vbyte ) .expect("failed to build transaction"); @@ -731,20 +785,17 @@ proptest! { .iter() .map(|utxo| (utxo.outpoint.clone(), utxo.value)) .collect(); - + let minter_address = BitcoinAddress::P2wpkhV0(main_pkhash); let (unsigned_tx, change_output, _) = build_unsigned_transaction( &mut utxos, vec![(BitcoinAddress::P2wpkhV0(dst_pkhash), target)], - BitcoinAddress::P2wpkhV0(main_pkhash), + minter_address.clone(), fee_per_vbyte ) .expect("failed to build transaction"); let fee = fake_sign(&unsigned_tx).vsize() as u64 * fee_per_vbyte / 1000; - let minter_fee = - crate::MINTER_FEE_PER_INPUT * unsigned_tx.inputs.len() as u64 + - crate::MINTER_FEE_PER_OUTPUT * unsigned_tx.outputs.len() as u64 + - MINTER_FEE_CONSTANT; + let minter_fee = evaluate_minter_fee(unsigned_tx.inputs.len() as u64, unsigned_tx.outputs.len() as u64); let inputs_value = unsigned_tx.inputs .iter() @@ -760,7 +811,7 @@ proptest! { }, tx::TxOut { value: inputs_value - target + minter_fee, - address: BitcoinAddress::P2wpkhV0(main_pkhash), + address: minter_address, }, ] ); diff --git a/rs/bitcoin/kyt/BUILD.bazel b/rs/bitcoin/kyt/BUILD.bazel index 11357e91738..456322064a7 100644 --- a/rs/bitcoin/kyt/BUILD.bazel +++ b/rs/bitcoin/kyt/BUILD.bazel @@ -67,8 +67,11 @@ rust_canister( "@crate_index//:futures", "@crate_index//:hex", "@crate_index//:ic-btc-interface", + "@crate_index//:ic-canister-log", "@crate_index//:ic-cdk", + "@crate_index//:ic-metrics-encoder", "@crate_index//:ic-stable-structures", + "@crate_index//:num-traits", "@crate_index//:serde", "@crate_index//:serde_json", "@crate_index//:time", @@ -94,6 +97,7 @@ rust_ic_test( ":btc_kyt_lib", "//:pocket-ic-server", "//packages/pocket-ic", + "//rs/rust_canisters/http_types", "//rs/test_utilities/load_wasm", "//rs/types/base_types", "//rs/types/types", diff --git a/rs/bitcoin/kyt/Cargo.toml b/rs/bitcoin/kyt/Cargo.toml index 81e1aa7a812..ea83efe59d8 100644 --- a/rs/bitcoin/kyt/Cargo.toml +++ b/rs/bitcoin/kyt/Cargo.toml @@ -19,9 +19,12 @@ ciborium = { workspace = true } futures = { workspace = true } hex = { workspace = true } ic-btc-interface = { workspace = true } +ic-canister-log = { path = "../../rust_canisters/canister_log" } ic-canisters-http-types = { path = "../../rust_canisters/http_types" } ic-cdk = { workspace = true } +ic-metrics-encoder = "1.1" ic-stable-structures = { workspace = true } +num-traits = { workspace = true } serde = { workspace = true } serde_json = {workspace = true } time = { workspace = true } diff --git a/rs/bitcoin/kyt/src/blocklist.rs b/rs/bitcoin/kyt/src/blocklist.rs index 3f27965b5b6..c798d0e0eb7 100644 --- a/rs/bitcoin/kyt/src/blocklist.rs +++ b/rs/bitcoin/kyt/src/blocklist.rs @@ -1,33 +1,43 @@ -/// The list of addresses to which we do not allow retrievals. +/// BTC is not accepted from nor sent to addresses on this list. /// NOTE: Keep it sorted! pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "123WBUDmSJv4GctdVEz6Qq6z8nXSKrJ4KX", + "125W5ek3DT6Zqy5S2iPt4FHQdNMCbZA3FU", "1295rkVyNfFpqZpXvKGhDqwhP1jZcNNDMV", "129zKFLoVad9JtxSmDKeJoLCsjhGR7b3vr", "12HQDsicffSBaYdJ6BhnE22sfjTESmmzKx", "12NpCkhddSNiDkD9rRYUCHsTT9ReMNiJjG", "12QtD5BFwRsdNsAZY76UVE1xyCGNTojH9h", "12VrYZgS1nmf9KHHped24xBb1aLLRpV2cT", + "12YCfVAEzkEZXBYhUTyJJaRkgMXiFxJgcu", "12YyR9EpvHxBjjKjTWqfKqeyoWnvcraxpW", + "12jVCWW1ZhTLA5yVnroEJswqKwsfiZKsax", "12mNKr2YP4M3CEQvCvVqZsvxuCG47LHMu1", "12udabs2TkX7NXCSj6KpqXfakjE52ZPLhz", + "12w6v1qAaBc4W8h8C2Cu5SKFaKDSv3erUW", "134r8iHv69xdT6p5qVKTsHrcUEuBVZAYak", + "13JtX4h7G5ZuNK5mFudKGq9DHLvvMFuNuz", "13LQJQ1oJ9K7PsqsGfjNhoVv6UeU6hgzQz", "13RH4JaFhaCxDGPyYE9emjp2aDxdX18uBA", "13ViCDZyJxxv5cZzpDDsE7aDQ3Y552zpAH", "13YBQr2Cp1YY3xqq2qngaPb7ca1o4ugeq6", "13f59kUM5FU8MfTG7DCEugYarDhSD7XCoC", + "13fhnkmpBBWXUQucJd6efWvXdEj78DKavk", "13hfsQm6oCaDZehfYBSMFiJVAi1jsL6sQd", "13mnk8SvDGqsQTHbiGiHBXqtaQCUKfcsnP", "148LKmyZT3FGE4x1GjsFN6RsAwcjzk5iuE", "149w62rY42aZBox8fGcmqNsXUzSStKeq8C", "14gM1HuLVDELNHaFU22qpabjtiWek4HhV1", "14kqryJUxM3a7aEi117KX9hoLUw592WsMR", + "14rjAD8ZP5xaL571cMRE98qgxxbg1S8mAN", "158treVZBGMBThoaympxccPdZPtqUfYrT9", "15PggTG7YhJKiE6B16vkKzA1YDTZipXEX4", "15Pt4NwZaUmMUwS2bQbyyncc7mzgWShtv8", "15UdZbmGPa2LatD3abtGpphgkHLFWftV4R", "15YK647qtoZQDzNrvY6HJL6QwXduLHfT28", + "15cRqR3TXS1JehBGWERuxFE8NhWZzfoeeU", + "15kZobLkD6HZgEECtz4oS2Vz21XHTnNfSg", + "15qyVrZvvVGvB7GWiAZ82TNcZ6QWMKu3kx", "15uqdxqNXQwVf5H7yZPz4TmEGeSccCwdor", "15yqWQ4sqr7jzCwDtZ3U1KaCa8WMEy7Mm2", "16EKTes8ahD8xvwisqjc2xSNLiG3fDHatW", @@ -36,6 +46,7 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "16ZSAEfYpPCj3D94fsNt2okYj9Ue8mxy6T", "16p2UWTZwXRyK5bTHNVjdDyy1D3EQGsZf2", "16tByCYzxuWiN8kF9FrK9jJy6eQYLVkQ1i", + "16ti2EXaae5izfkUZ1Zc59HMcsdnHpP5QJ", "175BUqf8JCU1uoG1iTRKTacDa4uvJDUCw2", "17UVSMegvrzfobKC82dHXpZLtLcqzW9stF", "17V7THwHMiDJmDwZK4unhE5HgKFJKx7VCe", @@ -45,14 +56,20 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "182NGZbPJXwg2WDrhrPpR7tpiGQkNPF844", "18Ke1QWE9nQfXuhJijHggZuPJ5ZYxapoBK", "18Qj1THHuETfYhuRDZycXJbWwDMGw73Poa", + "18aqbRhHupgvC9K8qEqD78phmTQQWs7B5d", "18cFGAdYcvNHkuhXLBE7izQKCyUW8TzCJE", + "18gaXypKj9M23S2zT9qZfL9iPbLFM372Q5", "18uKfaUjgG52rVeXEi3wxnveww7zZuECtE", + "18yWCu6agTxYqAerMxiz9sgHrK3ViezzGa", "194xmrZA53UBsZau2PnJLdmVVW9m5feeS", + "1986rYHckYbJpGQJy6ornuMyD2N5MTqwDt", + "19D1iGzDr7FyAdiy3ZZdxMd6ttHj1kj6WW", "19D8PHBjZH29uS1uPZ4m3sVyqqfF8UFG9o", "19FQzHibWDhSP8pKmJS3uagFYoisXtehzw", "19GrL5jnUkGmHXVcraB1Etv5rXCANeLWpq", "19pPbUDvoSBZafkUCYkD2Z9AkuqqV6sWm7", "1A3iYY4c3dkgNYGewzYzr7EsqfBuWXibGo", + "1A7SKE2dQtezLktCY8peLsdAtkqxV9r1dC", "1ANpca7g93BwptUJg1zV116v49zn9gjDi3", "1AXUTu9y3H8w4wYx4BjyFWgRhZKDhmcMrn", "1AeSq93WDNdLoEJ92sex7T8xQZoYYm8BtS", @@ -68,13 +85,16 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "1CF46Rfbp97absrs7zb7dFfZS6qBXUm9EP", "1CG1aSCxUnbmv9G34ofxTQoHtuVnMLJtQV", "1CNbhgxGRZvsWnEHotfXge7k2E1UPzBDC7", + "1CPJak9ZyddbawMGJPyEhCiJLXXb4sYv8N", "1CddRqw7oSPrT4tt5oXKyx2LiHJDPszy7y", "1D1ej7zQzywWBDNXKNYpmH7Hso2U9koDG4", "1D3GuaS9eqKw8dWj9JFQtNufdRtysjSLxZ", + "1D6gG9iKEhPitTcWRJiniuj1jYM2hNeAfJ", "1DDA93oZPn7wte2eR1ABwcFoxUFxkKMwCf", "1DGsY4ww3BJnWXTsnmTgWa6UWdoRXgA1pX", "1DH2xDH7TngrDU6LXciprKCBKNcPA1xX8A", "1DJoEMvp95yJYWyxAZy8DDBzuvjnrTVrsN", + "1DJoVLgn1foJHHngduRPJvRbwpaFEKxvxd", "1DKGRGJXGNLAtTeFb9SNPNHtrkZ87q7qKi", "1DT3tenf14cxz9WFNxmYrXFbB6TFiVWA9U", "1DYFJ6CuBvrxyoQSuBzVsNcetY9tvdsrag", @@ -95,6 +115,7 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "1F2Gdug9ib9NQMhKMGGJczzMk5SuENoqrp", "1F317n2eJEMaEMGiwCqtd5XCU3wF7jzPEW", "1F7UL41qYm6TvnExZzPHBCyeENvX3XDEMS", + "1FCNgfZWpYMeYhy9t18AAkTBu8AsdoAc1Z", "1FE2cuvkq8n5VGwj5hi8YYQxskwJpovPyV", "1FFS6pX1TCKTNy668Mbk2Lyoem1qB48kYX", "1FRyL9gmFGbzfYDAB4iY9836DJe3KSnjP9", @@ -117,15 +138,19 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "1Hn9ErTCPRP6j5UDBeuXPGuq5RtRjFJxJQ", "1Hpj6qm9i7nMF3VkKfBFtjhEDpEjxHWvgv", "1HuYfoEwsfHgZiRhbhJrCd5ST3iksa8KEx", + "1J378PbmTKn2sEw6NBrSWVfjZLBZW3DZem", "1J6cgUVEZRKyJhpXJgHWX7YmzkdnHRaLhF", "1J9oGoAiHeRfeMZeUnJ9W7RpV55CdKtgYE", "1J9wJH2bamZVxscXAvoDH4jvtGKb7sYFDm", + "1JHdQHkBZiim1cb4hyUh2PbzEbbg6z2TrF", + "1JPfVv5cWRLx1js9NWxg46dG2CGbeRz4th", "1JREJdZupiFhE7ZzQPtASuMCvvpXC7wRsC", "1Js6goCey2NaqPQptiLANLQGuk4d6mowjP", "1Juv2Ks3jJFUes8jEGxwgt6T6csBRQmmRw", "1K2fmE9hfhbRNSZoBvCBWZAvsS5idTUxBG", "1KSAbh5trMCTZwhiNsuUQvfTtSSTT8zqRk", "1KUUJPkyDhamZXgpsyXqNGc3x1QPXtdhgz", + "1Kc6egXevyLEaeTxLFA1Zyw7GuhCN8jQtt", "1KctQENEX5QkQMpnMC3Zh9yRAzkMBLpPcr", "1KgudqxMfYaGzqAA7MS4DcsqejtMteqhix", "1KkaKujnqwJf7Cbm7JKAZGF3X9d4685m8n", @@ -138,19 +163,24 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "1LQV6yUBcfTjAWvFu3XPhdTgjqihss7i1z", "1LiNmTUPSJEd92ZgVJjAV3RT9BzUjvUCkx", "1Licqjca74n8pmNaoARXLLqcTUTHFpxbXH", + "1Ljk8RNNabkZ9bfDYQBn98XfFozJhTjqcZ", "1LpYKb3SXZPve9hsH2QEJZFX279wJVGowi", "1LrxsRd7zNuxPJcL5rttnoeJFy1y4AffYY", "1LuDiMd95Df4i2bcvrfw47t2GKpLLXAQMZ", + "1MLPqwaFFUBZePTjpR3nxRoK19Cv9mPCc7", "1MQBDeRWsiJBf7K1VGjJ7PWEL6GJXMfmLg", "1MbtT2ZsTtLp7EKZUV9r74cTyqvsMtTP2M", "1MiQRekg4BatJ12qbiSGnNakLLd8xbLMCG", + "1MkCnCa9agS5t6V1B15bzusBgYECB4LfWp", "1MnbhWe5wr7Ut45ReyQsm96PwnM9jD7KaH", "1MtsQsw6n2jvJCWhpCw7jifTfD9Q3rBBVg", "1N6XqSf3ULpNjko9LrJmHudRoLitjwkETN", "1NE2NiGhhbkFPSEyNWwj7hKGhGDedBtSrQ", + "1NaRX1GZgtZ7E8iXo8YUdTtnb8rAK5QFJa", "1NayLEVF3bEEbDtdF2Cwso1VdEtvVNh2qX", "1NbGwQwt4uEhg2srAKppLf8QaF6fbp3PZG", "1NpHuti9NSM9fVTXLkvSDU4AnhqGQ5N53d", + "1NuBZQXJPyYQGfoBib8wWBDpZmbtkJa5Ba", "1NvJm3jfZxENNyqws5BKQvhkLxg9chLJdo", "1P3ZfGFLezzYGg9k5SVzQmnjyh7nrUmF2y", "1PJp8diNa89cVHpiT1VPu7EQ8LxYM5HX6v", @@ -176,6 +206,7 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "32fbAZMTaQxNd2fAue1PgsiPgWfcsHBQQt", "32jgFkZsTEjMFaBvxJnYvJEeTNKTmq5b32", "32pCmCWEjwhkLwh5BgLNAeBQFp5Gi1hv81", + "32pTjxTNi7snk8sodrgfmdKao3DEn1nVJM", "32wdqwX3zCEX3DhAVEcKwXCEGdzgBnx1R9", "331TS6DyASY7iU5CRA8UryBnkPS78fP2B1", "33KKjn4exdBJQkTtdWxqpdVsWxrw3LareG", @@ -201,6 +232,7 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "35vypiSvQsxRiT3YZzGRGVaduUSx67ysZb", "361AkMKNNWYwZRsCE8pPNmoh5aQf4V7g4p", "361NP7YcBPQ4KkLT3Y2QZeDEV4M3yi65Ar", + "3685sEusmTwZBiKJ4cgV73EAhpVD1nbgbe", "36XqYWGvUQwBrYLRVuegN4pJJJSPWL1WEu", "36YGN5dGzqrxMomTHdkT6cYVMnWBw8S7hD", "36yS87PLuW7sErLg1TY26WzaVarTim7AcC", @@ -210,6 +242,7 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "386wa1UM6nA798AWNh64jdrejZyedeXgUN", "389Sft4nJFkPGhbagk9FN4jXncA9piYTuU", "38LjCapRrJEW7w2zwbyS15P9D9UGPjWS44", + "38TftW1heYPqvKbiAhT4ttNBqkUPpooXad", "38ncxqt932N9CcfNfYuHGZgCyR85hDkWBW", "39AALn7eTjdPzLb99hHhD6F7J8QWB3R2Rd", "39KQvziHwUe2vddbpfC5WkQEV72qbQhxuh", @@ -217,6 +250,7 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "39Te8MbphSgs7npDJPj2hbNzhke61NTcnB", "39eboeqYNFe2VoLC3mUGx4dh6GNhLB3D2q", "39fhoB2DohisGBbHvvfmkdPdShT75CNHdX", + "39p8qWp1bkBNhi4vPpFTetKPtH7goqNDZf", "3A1HH3PseYMkh2nSrBb4kkVt3815kUNVVC", "3AFcE2mbSSndcpYFgHoExSmjUc26ef2gQh", "3ANWhUnHujdwbw2jEuGSRH6bvFsD9BqEy9", @@ -226,6 +260,7 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "3AjyprBY5yhijiCjUC5NUJutGbwhd3AQdE", "3B3vmabBbeDRnVrjvvq3hm85zVB4v5bWFC", "3B4G1M8eF3cThbeMwhEWkKzczw9QoNTGak", + "3B7S6zrgxQVQUHTU8wstM23tB9afE7ojuX", "3BCN3WgMRJwULTz1vsEQ7NZrBjwaUBf5Ca", "3BQACtiMXYB9JpUMpkEWt9m8BzswpGHq4X", "3BWP6ZQAhc4j5wR1b95zJAthJEFvhdees7", @@ -262,6 +297,7 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "3HSZc4BLnQBznjSq7JvXgqNCZUUs3M9fZz", "3HWjh69cVQvcPeLWVCyVmXEq72nyDSj5zP", "3He6EyDaCUgmdr4GXqhxbeTQukaGLCByU2", + "3HePkztPU6UsdENxnyQHFrdZA6eVKeMtvz", "3HqA7i3ttECLvgqvq69HNxxUP5BL7Z5YgA", "3HupEUfKmMhvhXqf8TMoPAyqDcRC1kpe65", "3J19qffPT6mxQUcV6k5yVURGZtdhpdGr4y", @@ -298,15 +334,19 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "3N6WeZ6i34taX8Ditser6LKWBcXmt2XXL4", "3N9YcPBDky9UsMx1RTk33tL4jDkZfSnsPk", "3NDzzVxiLBUs1WPvVGRfCYDTAD2Ua2PvW4", + "3NNcbBcMZokPB2JC2dxYuZrpgcetvyGeJn", "3NPognMSbzyA2JYW2fpkVKWyBMi2XTq2Zt", "3NQ1aa9ceirMJ1JvRq3eXefvXj1L639fzX", "3NRJ8aXdUiZdHaiFX9ePX3DhGHzcEi14Fq", + "3NU9zkD8CCVGVmYnNNSN7PbU9sSFQBm5UV", "3P6PzdfETr4275Gn3veLkCyDxA1jV8fHKm", "3PDmRwotTkRAFRLGTUrucCERp2JdM1q4ar", + "3PKiHs4GY4rFg8dpppNVPXGPqMX6K2cBML", "3PUmTuVAW3LkKg53FZ7F97VDBitW4ugwnM", "3PiCnZrBvGfWAKQ9hr4cCpfaDjy64yNSpE", "3PyzSbFj3hbQQjTzDzyLSgvFVDjB7yw4Cj", "3Q5dGfLKkWqWSwYtbMUyc8xGjN5LrRviK4", + "3Q8H2ZWMtc4R1M3mkmhnTjCoYKTeCFigDP", "3QAdoc1rDCt8dii1GVPJXvvK6CEJLzCRZw", "3QEjBiPzw6WZUL4MYMmMU6DY1Y25aVbpQu", "3QJyT8nThEQakbfqgX86YjCK1Sp9hfNCUW", @@ -315,30 +355,85 @@ pub const BTC_ADDRESS_BLOCKLIST: &[&str] = &[ "3QnWE5GVfQu3wVav91RuFkqip4Ti4NWqAY", "3QrukkUiBrn23rFUKUgasNd1wYWNk7WdSV", "3Qw9Fn19gCnga9LfHfpM99aGzuqxBNjR2i", + "TUCsTq7TofTCJRRoHk6RvhMoS2mJLm5Yzq", + "bc1q05aktddf9ce4p7hh3stgsf253m4vweu7nkhtmw", "bc1q202ajnhxgg9d9jjczmg0g4usp6haqldyy2eakl", "bc1q237mvl0heyw0r38wd3xz8h5mar96rrwpams8pp", + "bc1q2jys00x2rgdkm3xnewuucqacytu0a7echupu8y", "bc1q2lpgjntr348pfvxhfy33ehmdzy3gmx8w4052z6", + "bc1q2q8uxjznmurg363dvd98xjg54mrr7z6mw9t825", + "bc1q362njxy39gnwrvj7zytn84ax39fwnhvk7n2999", + "bc1q3ael5my37nklgnqcrkwmnndfx9qdnp67j0739c", "bc1q3y5v2khlyvemcz042wl98dzflywr8ghglqws6s", + "bc1q49ax787wv0mnn8wjgp3sx772qz7eun822rkv5k", + "bc1q4namcagg5wljs0u20z6h2sqgsq4q5lts8rt2rr", "bc1q4rzdtlt0uslyw86cp29sctl6ct29g9a95cuup7pn5md9ddj7xgmqpp5m73", + "bc1q5jqgm7nvrhaw2rh2vk0dk8e4gg5g373g0vz07r", + "bc1q5zd0cwzd09k9r8xfk68sn5ytqp8f5aae80yfsm", + "bc1q68uswkjvu7nj7mhrvfzkx7cm6u5s9puvpm5dc2", + "bc1q6w463k7mhsgguwgqtcratm4vm42rncwhvvkeck", + "bc1q7hhdygx05kyfyzjku2u7lywvx5pgyng6a0nefp", + "bc1q7xfqqw9htr88t7vafg80p4qrlpjfyaps452x5g", "bc1q86tl9255vg5wldamfymaaz36uqxzm30gs7fhkljvzdlt9t38s3lqgdwdfq", + "bc1q8ew45w2agdffrnwp6adt2gqrc9n4mkev9ns29c", + "bc1q8pw85s83mdx2f3rfu64mmfd9wexqu2y856arwp", + "bc1q8q7z3kt37z6jalv5ujung5lem6pzlk9r3kt66k", "bc1q90zrdysy4flyacw7hsury3ajs9yzwtwp6guqpypx94w0d3p58hysvz6pde", "bc1q9lvynkfpaw330uhqmunzdz6gmafsvapv7y3zty", + "bc1q9ua9ypl4dhj0zut5kzasj5c2kxclhh8v2k9djd", + "bc1q9ws2gcq7uumm4mk3l9xezwve7w5tmcs5js5cup", + "bc1qa2xr7dmz5lztplp9yfp7k382nf4ma8gwrl7zgg", + "bc1qagp0gy58v8hqvw4p2wsphcxg067rrppp45hexr", + "bc1qakycg2zp8hydg95lc9cheumpa9yfpdeyrqqh5f", + "bc1qardcxz845jw83vgfvcmewhuxa0ag9gchjwmcfd", + "bc1qau9pmgc8l6rxpdwwap3fd6zprh5yp9mpe9jl0h", + "bc1qavwamr74qlzj8txy6jaxqnpym9062h090x6kz4", + "bc1qdpazd6smrkq5hmw6lupq98za2fqtgmq3azv3av", "bc1qdt3gml5z5n50y5hm04u2yjdphefkm0fl2zdj68", + "bc1qe8wclszdtshkjk7gph57crc72vpp9rylujgwa5", "bc1qe95l438kzjcvnsm3kn8n5augf9gpctdlhsq7f7hpnkyvlr7rc7cqupapf7", + "bc1qe9lz50jq0a5pmtry0h3ekng3kjdg09vejg7355", + "bc1qedvcgh32dud85yt7fu6s7qavthlh38gtwa7rhx", "bc1qfg4gfg0y6t6xjnpmlhuwx5k0wlw6nmfzxn2psc", + "bc1qgffxa65gr579tsz55n87pal2a777ygzg69d2yz", + "bc1qh33xtpjqhgysq5xlmjzkm28uewj08885n95drr", + "bc1qh3lw22uqwrywr8lpun20q2ma528a4yqmgf3uqp", + "bc1qhvzl5w99m458nm7sc6hwf5qfhxndw28sgtatk8", "bc1qj6j6p0jdefl6pvdzx3kx8245yy5mz6q4luhzes", + "bc1qkfqh5p2qsakseg5gnwen6zh5xf3elmku2xaup29mw3l0wnncy3es880dss", + "bc1ql526s72gycmvq7hek6j3tuwwmcpv4rgs0auxu7", "bc1ql7dlyh8xz6tpqk92vztrhqh88dmjvcwrmsemrm", + "bc1ql7rnd70aejdxt3f9fzdlfazjlc9hvdmut8ad8s", + "bc1qlfpg2xn39u580cmwq6rymt8jdhcmj8520jhgh0", + "bc1qllmlvy5u038yvwywu803p53g8udcm8w7k4qxu9", + "bc1qn6segn8km4nfdp9vueu6msfjsaxaqgun9h60n9", + "bc1qnykkrlk67f9kdqvzyw2ndu3xmef5z2e52886yl", "bc1qpaly5nm7pfka9v92d6qvl4fc2l9xzee8a6ys3s", + "bc1qq7p0es3dv5hcynjjf40f2xjjr6qp5py47d2f6n847vduuq9gvnyq7y9ecd", "bc1qqf8kcc9m57xjqcvsvuf989nnl48ve6d2s24cx3", + "bc1qrc2gchg2fmxua5u7twu4luv5p9twmny4jjmg9x", + "bc1qrskwdw9unhlkt87ltcqq2d5pn9s6w2f35gz3z6", "bc1qs9u6j78e3utj08mwvqkkmqm9de5xk3g4yh8qtq", "bc1qsmqpalp3gtgkltag4x3ygevmhh9y2hzk73t2ug", "bc1qsmv6lkrw65l30yazdqpdjjtwzpvk9f8gfh0cy7", "bc1qsxf77cvwcd6jv6j8d8j3uhh4g0xqw4meswmwuc", + "bc1qtdupwe722vcc5e0vh94cgwfs0ep4gzwyfsf75q", + "bc1qte9ret332gwrk6e7vqc87d807npzvdj5savg4h", + "bc1qtrrfhr0f3ufsrjxfv8a7p3yuqj0spe4cm6vaws", + "bc1qtsl3ufenrv3zgrgm9z8xarcx22x8mfztuartyn", + "bc1qtywfzxx6snut2mdrum8dyr5nnd5qhqd29wmvqt", + "bc1qu5vngdtcx8nc9d68pq8nv7pzcjrswsy87s66gk", + "bc1qup656lpfqfckhl580kwf62thmn5azmj2pal0sz", "bc1quyc6j8ca84q9gjej5jjd2n8hra0vfu0j60fefs57p6e5rerkq07q0l5u3w", + "bc1qv4krhj2qmpd9qz9xj9nhvj99fq8s9xsw05d485", "bc1qv7k70u2zynvem59u88ctdlaw7hc735d8xep9rq", + "bc1qvhnfknw852ephxyc5hm4q520zmvf9maphetc9z", "bc1qvlzfn6kmezv44d8kw0p5jsmxe6wchv3zc7gsxs", "bc1qw4cxpe6sxa5dg6sdwxjph959cw6yztrzl4r54s", "bc1qw7vfgv3r5vnehafl0y95sclg3uqsj87wxs9ad628yjjcq33cwessr6ndyw", + "bc1qw8evftpgc8wsmkemd8yl5fg2husza5z802xaym", "bc1qwa6zu6qhl6wqnlxp642vcf89nptsassle25ulf", + "bc1qwz0l2ceddckwdy5rh6zyrm3x4ha6gn5f2da5zg", "bc1qx3e2axj3wsfn0ndtvlwmkghmmgm4583nqg8ngk", + "bc1qx9upga7f09tsetqf78wa3qrmcjar58mkwz6ng6", ]; diff --git a/rs/bitcoin/kyt/src/fetch.rs b/rs/bitcoin/kyt/src/fetch.rs index 4a7b208f8c1..81f011f6be0 100644 --- a/rs/bitcoin/kyt/src/fetch.rs +++ b/rs/bitcoin/kyt/src/fetch.rs @@ -1,3 +1,4 @@ +use crate::logs::WARN; use crate::state::{ FetchGuardError, FetchTxStatus, FetchTxStatusError, FetchedTx, HttpGetTxError, TransactionKytData, @@ -11,6 +12,7 @@ use ic_btc_kyt::{ CheckTransactionRetriable, CheckTransactionStatus, INITIAL_MAX_RESPONSE_BYTES, RETRY_MAX_RESPONSE_BYTES, }; +use ic_canister_log::log; use std::convert::Infallible; #[cfg(test)] @@ -216,14 +218,13 @@ pub trait FetchEnv { state::set_fetched_address(txid, index, address.clone()); } else { // This error shouldn't happen unless blockdata is corrupted. - // TODO(XC-205): log this error - return CheckTransactionIrrecoverableError::InvalidTransaction( - format!( - "Tx {} vout {} has no address, but is vin {} of tx {}", - input.txid, input.vout, index, txid - ), - ) - .into(); + let msg = format!( + "Tx {} vout {} has no address, but is vin {} of tx {}", + input.txid, input.vout, index, txid + ); + log!(WARN, "{msg}"); + return CheckTransactionIrrecoverableError::InvalidTransaction(msg) + .into(); } } Pending => {} @@ -259,13 +260,13 @@ pub trait FetchEnv { state::set_fetched_address(txid, index, address.clone()); } else { // This error shouldn't happen unless blockdata is corrupted. - // TODO(XC-205): log this error + let msg = format!( + "Tx {} vout {} has no address, but is vin {} of tx {}", + input_txid, vout, index, txid + ); + log!(WARN, "{msg}"); error = Some( - CheckTransactionIrrecoverableError::InvalidTransaction(format!( - "Tx {} vout {} has no address, but is vin {} of tx {}", - input_txid, vout, index, txid - )) - .into(), + CheckTransactionIrrecoverableError::InvalidTransaction(msg).into(), ); } } diff --git a/rs/bitcoin/kyt/src/logs.rs b/rs/bitcoin/kyt/src/logs.rs new file mode 100644 index 00000000000..e6eba8ff83d --- /dev/null +++ b/rs/bitcoin/kyt/src/logs.rs @@ -0,0 +1,29 @@ +use ic_canister_log::declare_log_buffer; +use serde::{Deserialize, Serialize}; + +// High-priority messages. +declare_log_buffer!(name = WARN, capacity = 1000); + +// Low-priority info messages. +declare_log_buffer!(name = DEBUG, capacity = 1000); + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum Priority { + Warn, + Debug, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct LogEntry { + pub timestamp: u64, + pub priority: Priority, + pub file: String, + pub line: u32, + pub message: String, + pub counter: u64, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub struct Log { + pub entries: Vec, +} diff --git a/rs/bitcoin/kyt/src/main.rs b/rs/bitcoin/kyt/src/main.rs index 5517f36583e..92c29cf88e4 100644 --- a/rs/bitcoin/kyt/src/main.rs +++ b/rs/bitcoin/kyt/src/main.rs @@ -6,19 +6,37 @@ use ic_btc_kyt::{ CheckTransactionRetriable, CheckTransactionStatus, KytArg, KytMode, CHECK_TRANSACTION_CYCLES_REQUIRED, CHECK_TRANSACTION_CYCLES_SERVICE_FEE, }; +use ic_canister_log::{export as export_logs, log}; use ic_canisters_http_types as http; use ic_cdk::api::call::RejectionCode; use ic_cdk::api::management_canister::http_request::{HttpResponse, TransformArgs}; +use num_traits::cast::ToPrimitive; +use std::cell::RefCell; +use std::collections::BTreeMap; use std::str::FromStr; mod dashboard; mod fetch; +mod logs; mod providers; mod state; use fetch::{FetchEnv, FetchResult, TryFetchResult}; +use logs::{Log, LogEntry, Priority, DEBUG, WARN}; use state::{get_config, set_config, Config, FetchGuardError, HttpGetTxError}; +#[derive(Default)] +struct Stats { + https_outcall_status: BTreeMap<(String, u16), u64>, + http_response_size: BTreeMap, + check_transaction_count: u64, + check_address_count: u64, +} + +thread_local! { + static STATS : RefCell = RefCell::default(); +} + pub fn is_response_too_large(code: &RejectionCode, message: &str) -> bool { code == &RejectionCode::SysFatal && (message.contains("size limit") || message.contains("length limit")) @@ -38,6 +56,7 @@ fn check_address(args: CheckAddressArgs) -> CheckAddressResponse { ic_cdk::trap(&format!("Not a bitcoin {} address: {}", btc_network, err)) }); + STATS.with(|s| s.borrow_mut().check_transaction_count += 1); match config.kyt_mode() { KytMode::AcceptAll => CheckAddressResponse::Passed, KytMode::RejectAll => CheckAddressResponse::Failed, @@ -72,6 +91,7 @@ async fn check_transaction(args: CheckTransactionArgs) -> CheckTransactionRespon ic_cdk::api::call::msg_cycles_accept128(CHECK_TRANSACTION_CYCLES_SERVICE_FEE); match Txid::try_from(args.txid.as_ref()) { Ok(txid) => { + STATS.with(|s| s.borrow_mut().check_transaction_count += 1); if ic_cdk::api::call::msg_cycles_available128() .checked_add(CHECK_TRANSACTION_CYCLES_SERVICE_FEE) .unwrap() @@ -126,9 +146,145 @@ fn post_upgrade(arg: KytArg) { #[ic_cdk::query(hidden = true)] fn http_request(req: http::HttpRequest) -> http::HttpResponse { + if ic_cdk::api::data_certificate().is_none() { + ic_cdk::trap("update call rejected"); + } + + #[cfg(target_arch = "wasm32")] + fn heap_memory_size_bytes() -> usize { + const WASM_PAGE_SIZE_BYTES: usize = 65536; + core::arch::wasm32::memory_size(0) * WASM_PAGE_SIZE_BYTES + } + + #[cfg(not(any(target_arch = "wasm32")))] + fn heap_memory_size_bytes() -> usize { + 0 + } + if req.path() == "/metrics" { - // TODO(XC-205): Add metrics - unimplemented!() + let mut writer = + ic_metrics_encoder::MetricsEncoder::new(vec![], ic_cdk::api::time() as i64 / 1_000_000); + + let cycle_balance = ic_cdk::api::canister_balance128() as f64; + + writer + .gauge_vec("cycle_balance", "The canister cycle balance.") + .unwrap() + .value(&[("canister", "btc-kyt")], cycle_balance) + .unwrap(); + + writer + .encode_gauge( + "heap_memory_bytes", + heap_memory_size_bytes() as f64, + "Size of the heap memory allocated by this canister.", + ) + .unwrap(); + + writer + .encode_gauge( + "stable_memory_bytes", + ic_cdk::api::stable::stable_size() as f64 * 65536.0, + "Size of the stable memory allocated by this canister.", + ) + .unwrap(); + + STATS.with(|s| { + let stats = s.borrow(); + let mut counter = writer + .counter_vec( + "btc_kyt_http_calls_total", + "The number of http outcalls made since the last canister upgrade.", + ) + .unwrap(); + for ((provider, status), count) in stats.https_outcall_status.iter() { + counter = counter + .value( + &[ + ("provider", provider.as_str()), + ("status", status.to_string().as_str()), + ], + *count as f64, + ) + .unwrap(); + } + let mut counter = writer + .counter_vec( + "btc_kyt_http_response_size", + "The byte sizes of http outcall responses.", + ) + .unwrap(); + for (size, count) in stats.http_response_size.iter() { + counter = counter + .value(&[("size", size.to_string().as_str())], *count as f64) + .unwrap(); + } + writer + .counter_vec( + "ckbtc_kyt_requests_total", + "The number of KYT requests received since the last canister upgrade.", + ) + .unwrap() + .value( + &[("type", "check_transaction")], + stats.check_transaction_count as f64, + ) + .unwrap() + .value( + &[("type", "check_address")], + stats.check_address_count as f64, + ) + .unwrap(); + }); + + http::HttpResponseBuilder::ok() + .header("Content-Type", "text/plain; version=0.0.4") + .with_body_and_content_length(writer.into_inner()) + .build() + } else if req.path() == "/logs" { + use serde_json; + + let max_skip_timestamp = match req.raw_query_param("time") { + Some(arg) => match u64::from_str(arg) { + Ok(value) => value, + Err(_) => { + return http::HttpResponseBuilder::bad_request() + .with_body_and_content_length("failed to parse the 'time' parameter") + .build() + } + }, + None => 0, + }; + + let mut entries: Log = Default::default(); + for entry in export_logs(&WARN) { + if entry.timestamp >= max_skip_timestamp { + entries.entries.push(LogEntry { + timestamp: entry.timestamp, + counter: entry.counter, + priority: Priority::Warn, + file: entry.file.to_string(), + line: entry.line, + message: entry.message, + }); + } + } + for entry in export_logs(&DEBUG) { + if entry.timestamp >= max_skip_timestamp { + entries.entries.push(LogEntry { + timestamp: entry.timestamp, + counter: entry.counter, + priority: Priority::Debug, + file: entry.file.to_string(), + line: entry.line, + message: entry.message, + }); + } + } + http::HttpResponseBuilder::ok() + .header("Content-Type", "application/json; charset=utf-8") + .with_body_and_content_length(serde_json::to_string(&entries).unwrap_or_default()) + .build() } else if req.path() == "/dashboard" { use askama::Template; let page_index = match req.raw_query_param("page") { @@ -182,6 +338,18 @@ impl FetchEnv for KytCanisterEnv { let cycles = get_tx_cycle_cost(max_response_bytes); match http_request(request.clone(), cycles).await { Ok((response,)) => { + STATS.with(|s| { + let mut stat = s.borrow_mut(); + *stat + .https_outcall_status + .entry((provider.name(), response.status.0.to_u16().unwrap())) + .or_default() += 1; + // Calculate size bucket as a series of power of 2s. + // Note that the max is bounded by `max_response_bytes`, which fits `u32`. + let size = 2u32.pow((response.body.len() as f64).log2().floor() as u32); + *stat.http_response_size.entry(size).or_default() += 1; + }); + // Ensure response is 200 before decoding if response.status != 200u32 { // All non-200 status are treated as transient errors @@ -224,8 +392,10 @@ impl FetchEnv for KytCanisterEnv { } Err((r, m)) if is_response_too_large(&r, &m) => Err(HttpGetTxError::ResponseTooLarge), Err((r, m)) => { - // TODO(XC-158): maybe try other providers and also log the error. - println!("The http_request resulted into error. RejectionCode: {r:?}, Error: {m}"); + log!( + DEBUG, + "The http_request resulted into error. RejectionCode: {r:?}, Error: {m}, Request: {request:?}" + ); Err(HttpGetTxError::Rejected { code: r, message: m, diff --git a/rs/bitcoin/kyt/src/providers.rs b/rs/bitcoin/kyt/src/providers.rs index c034b4b2e0c..51496544ae2 100644 --- a/rs/bitcoin/kyt/src/providers.rs +++ b/rs/bitcoin/kyt/src/providers.rs @@ -57,6 +57,14 @@ impl Provider { &self.btc_network } + pub fn name(&self) -> String { + match self.btc_network { + BtcNetwork::Mainnet => self.provider_id.to_string(), + BtcNetwork::Testnet => "Testnet".to_string(), + BtcNetwork::Regtest { .. } => "Regtest".to_string(), + } + } + // Return the next provider by cycling through all available providers. pub fn next(&self) -> Self { let btc_network = &self.btc_network; diff --git a/rs/bitcoin/kyt/tests/tests.rs b/rs/bitcoin/kyt/tests/tests.rs index dd39902019e..d85f48d617b 100644 --- a/rs/bitcoin/kyt/tests/tests.rs +++ b/rs/bitcoin/kyt/tests/tests.rs @@ -649,3 +649,48 @@ fn tick_until_next_request(env: &PocketIc) -> Vec { ); canister_http_requests } + +#[test] +fn should_query_logs_and_metrics() { + use candid::Decode; + + let setup = Setup::new(BtcNetwork::Mainnet); + test_http_query(&setup, "/metrics"); + test_http_query(&setup, "/logs"); + + fn test_http_query>(setup: &Setup, url: U) { + let request = ic_canisters_http_types::HttpRequest { + method: "GET".to_string(), + url: url.into(), + headers: Default::default(), + body: Default::default(), + }; + + let response = Decode!( + &assert_reply( + setup + .env + .query_call( + setup.kyt_canister, + Principal::anonymous(), + "http_request", + Encode!(&request).expect("failed to encode HTTP request"), + ) + .expect("failed to query get_transactions on the ledger") + ), + ic_canisters_http_types::HttpResponse + ) + .unwrap(); + + assert_eq!(response.status_code, 200_u16); + } +} + +fn assert_reply(result: WasmResult) -> Vec { + match result { + WasmResult::Reply(bytes) => bytes, + WasmResult::Reject(reject) => { + panic!("Expected a successful reply, got a reject: {}", reject) + } + } +} diff --git a/rs/boundary_node/anonymization/backend/Cargo.toml b/rs/boundary_node/anonymization/backend/Cargo.toml index aad6ba5a5bd..875e32b65c2 100644 --- a/rs/boundary_node/anonymization/backend/Cargo.toml +++ b/rs/boundary_node/anonymization/backend/Cargo.toml @@ -1,11 +1,11 @@ [package] -name = "anonymization-backend" +name = "anonymization_backend" version = "0.1.0" edition = "2021" [[bin]] -name = "anonymization-backend" -path = "src/lib.rs" +name = "anonymization_backend" +path = "src/main.rs" [dependencies] anyhow = { workspace = true } diff --git a/rs/boundary_node/anonymization/backend/src/lib.rs b/rs/boundary_node/anonymization/backend/src/main.rs similarity index 84% rename from rs/boundary_node/anonymization/backend/src/lib.rs rename to rs/boundary_node/anonymization/backend/src/main.rs index 391a8f8de30..c659c0b5bf5 100644 --- a/rs/boundary_node/anonymization/backend/src/lib.rs +++ b/rs/boundary_node/anonymization/backend/src/main.rs @@ -6,7 +6,10 @@ use anonymization_interface::{ SubmitResponse, }; use candid::Principal; -use ic_cdk::{id, spawn, trap}; +use ic_cdk::{ + api::{call::accept_message, stable::WASM_PAGE_SIZE_IN_BYTES, time}, + caller, id, spawn, trap, +}; use ic_cdk_timers::set_timer_interval; use ic_nns_constants::REGISTRY_CANISTER_ID; use ic_stable_structures::{ @@ -97,6 +100,20 @@ thread_local! { ).unwrap() }); + static GAUGE_CANISTER_STABLE_BYTES_TOTAL: RefCell = RefCell::new({ + Gauge::new( + format!("{SERVICE_NAME}_canister_stable_bytes_total"), // name + "stable memory byte size", // help + ).unwrap() + }); + + static GAUGE_CANISTER_LAST_UPDATE_SECS: RefCell = RefCell::new({ + Gauge::new( + format!("{SERVICE_NAME}_last_update_secs"), // name + "timestamp in seconds of the last canister update", // help + ).unwrap() + }); + static METRICS_REGISTRY: RefCell = RefCell::new({ let r = Registry::new(); @@ -135,6 +152,16 @@ thread_local! { r.register(g).unwrap(); }); + GAUGE_CANISTER_STABLE_BYTES_TOTAL.with(|g| { + let g = Box::new(g.borrow().to_owned()); + r.register(g).unwrap(); + }); + + GAUGE_CANISTER_LAST_UPDATE_SECS.with(|g| { + let g = Box::new(g.borrow().to_owned()); + r.register(g).unwrap(); + }); + r }); } @@ -224,9 +251,12 @@ thread_local! { // Timers +const SECOND: Duration = Duration::from_secs(1); +const DAY: Duration = Duration::from_secs(24 * 60 * 60); + fn timers() { // ACLs - set_timer_interval(Duration::from_secs(10), || { + set_timer_interval(10 * SECOND, || { // Switch to async spawn(async { // List registry entries @@ -244,6 +274,10 @@ fn timers() { // Clear allowed principals ps.clear_new(); + // Self authorize + ps.insert(id(), ()); + + // Authorize API Boundary Nodes ids.iter().for_each(|p| { ps.insert(p.to_owned(), ()); }); @@ -293,8 +327,33 @@ fn timers() { }); }); + // TTLs + set_timer_interval(7 * DAY, || { + // Remove all encrypted values + let ids = ENCRYPTED_VALUES.with(|vs| { + let mut vs = vs.borrow_mut(); + + let ids: Vec<_> = vs.iter().map(|(k, _)| k).collect(); + vs.clear_new(); + + ids + }); + + // Re-queue + QUEUE.with(|q| { + let mut q = q.borrow_mut(); + + for id in ids { + q.insert( + id, // principal + (), // unit + ); + } + }); + }); + // Leader - set_timer_interval(Duration::from_secs(30), || { + set_timer_interval(30 * SECOND, || { // Collect candidates that have registered a public-key let ps: Vec = PUBLIC_KEYS.with(|ks| ks.borrow().iter().map(|(p, _)| p).collect()); @@ -335,22 +394,30 @@ fn main() {} #[ic_cdk::init] fn init(_arg: InitArg) { - // Self-authorize - ALLOWED_PRINCIPALS.with(|m| { - m.borrow_mut().insert( - id(), // canister id - (), // unit - ) - }); - // Start timers timers(); + + // Set update time + GAUGE_CANISTER_LAST_UPDATE_SECS.with(|g| g.borrow_mut().set((time() as f64 / 1e9).trunc())); } #[ic_cdk::post_upgrade] fn post_upgrade() { // Start timers timers(); + + // Set update time + GAUGE_CANISTER_LAST_UPDATE_SECS.with(|g| g.borrow_mut().set((time() as f64 / 1e9).trunc())); +} + +#[ic_cdk::inspect_message] +fn inspect_message() { + (&AUTHORIZER) + .authorize(&caller()) + .err() + .inspect(|err| trap(&err.to_string())); + + accept_message(); } #[ic_cdk::update] @@ -428,6 +495,11 @@ fn http_request(request: HttpRequest) -> HttpResponse { GAUGE_CANISTER_CYCLES_BALANCE .with(|g| g.borrow_mut().set(ic_cdk::api::canister_balance() as f64)); + GAUGE_CANISTER_STABLE_BYTES_TOTAL.with(|g| { + g.borrow_mut() + .set((ic_cdk::api::stable::stable_size() * WASM_PAGE_SIZE_IN_BYTES) as f64) + }); + // Export metrics let bs = METRICS_REGISTRY.with(|r| { let mfs = r.borrow().gather(); diff --git a/rs/boundary_node/anonymization/client/BUILD.bazel b/rs/boundary_node/anonymization/client/BUILD.bazel new file mode 100644 index 00000000000..9e79c57884d --- /dev/null +++ b/rs/boundary_node/anonymization/client/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +DEPENDENCIES = [ + "//rs/boundary_node/anonymization/interface", + "//rs/canister_client", + "//rs/types/types", + "@crate_index//:anyhow", + "@crate_index//:candid", + "@crate_index//:rand", + "@crate_index//:rsa", + "@crate_index//:thiserror", + "@crate_index//:tokio", +] + +MACRO_DEPENDENCIES = [ + "@crate_index//:async-trait", +] + +rust_library( + name = "client", + srcs = glob(["src/**/*.rs"]), + crate_name = "anonymization_client", + proc_macro_deps = MACRO_DEPENDENCIES, + version = "0.1.0", + deps = DEPENDENCIES, +) diff --git a/rs/boundary_node/anonymization/client/src/lib.rs b/rs/boundary_node/anonymization/client/src/lib.rs index f5c87e7a5d2..16a614ceab8 100644 --- a/rs/boundary_node/anonymization/client/src/lib.rs +++ b/rs/boundary_node/anonymization/client/src/lib.rs @@ -61,6 +61,14 @@ impl WithThrottle { } } +#[async_trait] +impl Register for WithThrottle { + async fn register(&self, pubkey: &[u8]) -> Result<(), RegisterError> { + self.throttle().await; + self.0.register(pubkey).await + } +} + #[async_trait] impl Query for WithThrottle { async fn query(&self) -> Result, QueryError> { @@ -421,7 +429,8 @@ impl From for CanisterMethods { Self { register: Arc::new({ let v = value.clone(); - WithLogs(v) + let v = WithLogs(v); + WithThrottle(v, ThrottleParams::new(Duration::from_secs(10))) }), query: Arc::new({ let v = value.clone(); diff --git a/rs/boundary_node/anonymization/interface/BUILD.bazel b/rs/boundary_node/anonymization/interface/BUILD.bazel new file mode 100644 index 00000000000..3b10d693316 --- /dev/null +++ b/rs/boundary_node/anonymization/interface/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +DEPENDENCIES = [ + # Keep sorted. + "@crate_index//:candid", + "@crate_index//:serde", + "@crate_index//:serde_bytes", +] + +rust_library( + name = "interface", + srcs = glob(["src/**/*.rs"]), + crate_name = "anonymization_interface", + version = "0.1.0", + deps = DEPENDENCIES, +) diff --git a/rs/boundary_node/ic_boundary/BUILD.bazel b/rs/boundary_node/ic_boundary/BUILD.bazel index 991402654a1..df3c3bf81a1 100644 --- a/rs/boundary_node/ic_boundary/BUILD.bazel +++ b/rs/boundary_node/ic_boundary/BUILD.bazel @@ -5,12 +5,18 @@ package(default_visibility = ["//visibility:public"]) DEPENDENCIES = [ # Keep sorted. + "//rs/boundary_node/anonymization/client", + "//rs/canister_client", + "//rs/canister_client/sender", "//rs/certification/test-utils", "//rs/config", + "//rs/crypto", "//rs/crypto/ed25519", "//rs/crypto/tree_hash", + "//rs/crypto/utils/basic_sig", "//rs/crypto/utils/threshold_sig_der", "//rs/crypto/utils/tls", + "//rs/interfaces", "//rs/interfaces/registry", "//rs/limits", "//rs/monitoring/logger", @@ -42,6 +48,7 @@ DEPENDENCIES = [ "@crate_index//:http", "@crate_index//:http-body", "@crate_index//:humantime", + "@crate_index//:ic-agent", "@crate_index//:ic-bn-lib", "@crate_index//:lazy_static", "@crate_index//:little-loadshedder", diff --git a/rs/boundary_node/ic_boundary/Cargo.toml b/rs/boundary_node/ic_boundary/Cargo.toml index 130d5e91bcf..5325dffe7a6 100644 --- a/rs/boundary_node/ic_boundary/Cargo.toml +++ b/rs/boundary_node/ic_boundary/Cargo.toml @@ -16,6 +16,7 @@ tls = [] [dependencies] anyhow = { workspace = true } +anonymization-client = { path = "../anonymization/client" } arc-swap = "1.7.1" async-scoped = { version = "0.9", features = ["use-tokio"] } async-trait = { workspace = true } @@ -32,14 +33,20 @@ hex = { workspace = true } http = { workspace = true } http-body = { workspace = true } humantime = "2.1" +ic-agent = "0.39.1" ic-base-types = { path = "../../types/base_types" } -ic-bn-lib = { git = "https://github.com/dfinity/ic-bn-lib", rev = "2496803db52d8182f578f098a3ffe4fcb9573fcd" } +ic-bn-lib = { git = "https://github.com/dfinity/ic-bn-lib", rev = "526d34d15cfbf369d8baf2dae9932aa18d570a1d" } +ic-canister-client = { path = "../../canister_client" } +ic-canister-client-sender = { path = "../../canister_client/sender" } ic-certification-test-utils = { path = "../../certification/test-utils" } ic-config = { path = "../../config" } +ic-crypto = { path = "../../crypto" } ic-crypto-ed25519 = { path = "../../crypto/ed25519" } ic-crypto-tree-hash = { path = "../../crypto/tree_hash" } +ic-crypto-utils-basic-sig = { path = "../../crypto/utils/basic_sig" } ic-crypto-utils-threshold-sig-der = { path = "../../crypto/utils/threshold_sig_der" } ic-crypto-utils-tls = { path = "../../crypto/utils/tls" } +ic-interfaces = { path = "../../interfaces" } ic-interfaces-registry = { path = "../../interfaces/registry" } ic-limits = { path = "../../limits" } ic-logger = { path = "../../monitoring/logger" } diff --git a/rs/boundary_node/ic_boundary/src/cli.rs b/rs/boundary_node/ic_boundary/src/cli.rs index 911a07e3c1d..b4aa03092b9 100644 --- a/rs/boundary_node/ic_boundary/src/cli.rs +++ b/rs/boundary_node/ic_boundary/src/cli.rs @@ -1,3 +1,4 @@ +use candid::Principal; use clap::{Args, Parser}; use humantime::parse_duration; use ic_bn_lib::{ @@ -8,6 +9,7 @@ use ic_bn_lib::{ parse_size, types::RequestType, }; +use ic_config::crypto::CryptoConfig; use std::time::Duration; use std::{net::SocketAddr, path::PathBuf}; use url::Url; @@ -228,6 +230,10 @@ pub struct Observability { /// Enables logging to /dev/null (to benchmark logging) #[clap(env, long)] pub obs_log_null: bool, + + /// Log Anonymization Canister ID + #[clap(env, long)] + pub obs_log_anonymization_canister_id: Option, } #[derive(Args)] @@ -366,4 +372,12 @@ pub struct Misc { /// Skip replica TLS certificate verification. DANGER: to be used only for testing #[clap(env, long)] pub skip_replica_tls_verification: bool, + + /// Configuration of the node's crypto-vault + #[clap(env, long, value_parser=parse_crypto_config)] + pub crypto_config: Option, +} + +fn parse_crypto_config(arg: &str) -> Result { + serde_json::from_str(arg) } diff --git a/rs/boundary_node/ic_boundary/src/core.rs b/rs/boundary_node/ic_boundary/src/core.rs index 3f286a55197..e185add1881 100644 --- a/rs/boundary_node/ic_boundary/src/core.rs +++ b/rs/boundary_node/ic_boundary/src/core.rs @@ -6,6 +6,9 @@ use std::{ time::{Duration, Instant}, }; +use anonymization_client::{ + Canister as AnonymizationCanister, Track, Tracker as AnonymizationTracker, +}; use anyhow::{anyhow, Context, Error}; use arc_swap::ArcSwapOption; use async_scoped::TokioScope; @@ -31,13 +34,19 @@ use ic_bn_lib::{ }, types::RequestType, }; +use ic_canister_client::{Agent, Sender}; +use ic_crypto::CryptoComponent; +use ic_crypto_utils_basic_sig::conversions::derive_node_id; +use ic_interfaces::crypto::{BasicSigner, KeyManager}; use ic_interfaces_registry::ZERO_REGISTRY_VERSION; -use ic_registry_client::client::RegistryClientImpl; -use ic_registry_local_store::{LocalStoreImpl, LocalStoreReader}; +use ic_logger::replica_logger::no_op_logger; +use ic_registry_client::client::{RegistryClient, RegistryClientImpl}; +use ic_registry_local_store::{LocalStore, LocalStoreImpl}; use ic_registry_replicator::RegistryReplicator; -use ic_types::crypto::threshold_sig::ThresholdSigPublicKey; +use ic_types::{crypto::threshold_sig::ThresholdSigPublicKey, messages::MessageId}; use nix::unistd::{getpgid, setpgid, Pid}; use prometheus::Registry; +use rand::rngs::OsRng; use tokio::sync::RwLock; use tokio_util::sync::CancellationToken; use tower::{limit::ConcurrencyLimitLayer, util::MapResponseLayer, ServiceBuilder}; @@ -210,6 +219,9 @@ pub async fn main(cli: Cli) -> Result<(), Error> { cli.rate_limiting.rate_limit_generic.clone(), )); + // HTTP Logs Anonymization + let anonymization_salt = Arc::new(ArcSwapOption::>::empty()); + // Prepare Axum Router let router = setup_router( registry_snapshot.clone(), @@ -220,6 +232,7 @@ pub async fn main(cli: Cli) -> Result<(), Error> { &cli, &metrics_registry, cache.clone(), + anonymization_salt.clone(), ); // HTTP server metrics @@ -315,36 +328,171 @@ pub async fn main(cli: Cli) -> Result<(), Error> { let persister = Persister::new(Arc::clone(&routing_table)); - let (registry_replicator, nns_pub_key, mut registry_runners) = - // Set up registry-related stuff if local store was specified - if cli.registry.registry_local_store_path.is_some() { - let RegistrySetupResult(registry_replicator, nns_pub_key, registry_runners) = - setup_registry( - &cli, - registry_snapshot.clone(), - WithMetricsPersist(persister, MetricParamsPersist::new(&metrics_registry)), - http_client_check, - &metrics_registry, - )?; - - (registry_replicator, nns_pub_key, registry_runners) - } else { - // Otherwise load a stub routing table and snapshot - let subnet = generate_stub_subnet(cli.registry.registry_stub_replica.clone()); - let snapshot = generate_stub_snapshot(vec![subnet.clone()]); - let _ = persister.persist(vec![subnet]); - registry_snapshot.store(Some(Arc::new(snapshot))); + if cli.registry.registry_local_store_path.is_none() { + // Prepare a stub routing table and snapshot + let subnet = generate_stub_subnet(cli.registry.registry_stub_replica.clone()); + let snapshot = generate_stub_snapshot(vec![subnet.clone()]); + let _ = persister.persist(vec![subnet]); + registry_snapshot.store(Some(Arc::new(snapshot))); + } - (None, None, vec![]) - }; + // Registry Client + let ( + registry_client, // + registry_replicator, // + nns_pub_key, // + mut registry_runners, // + ) = cli + .registry + .registry_local_store_path + .clone() + .map(|p| { + // Store + let local_store = Arc::new(LocalStoreImpl::new(p)); + + // Client + let registry_client = Arc::new(RegistryClientImpl::new( + local_store.clone(), // data_provider + None, // metrics_registry + )); + + registry_client + .fetch_and_start_polling() + .context("failed to start registry client")?; + + // Snapshotting + let r = setup_registry( + &cli, + local_store.clone(), + registry_client.clone(), + registry_snapshot.clone(), + WithMetricsPersist( + persister, // T + MetricParamsPersist::new(&metrics_registry), // Params + ), + http_client_check, + &metrics_registry, + )?; + + let RegistrySetupResult( + registry_replicator, // + nns_pub_key, // + registry_runners, // + ) = r; + + Ok::<_, Error>(( + registry_client, + registry_replicator, + nns_pub_key, + registry_runners, + )) + }) + .transpose()? + .map_or((None, None, None, vec![]), |(a, b, c, d)| { + (Some(a), b, c, d) + }); let generic_limiter_runner = WithThrottle(generic_limiter, ThrottleParams::new(10 * SECOND)); // Runners - let mut runners: Vec> = - vec![Box::new(metrics_runner), Box::new(generic_limiter_runner)]; + let mut runners: Vec> = vec![ + Box::new(metrics_runner), // + Box::new(generic_limiter_runner), // + ]; + runners.append(&mut registry_runners); + // HTTP Logs Anonymization + let tracker = match ( + registry_client, + cli.misc.crypto_config, + cli.obs.obs_log_anonymization_canister_id, + ) { + (Some(registry_client), Some(crypto_config), Some(log_anonymization_cid)) => { + let c = tokio::task::spawn_blocking({ + let registry_client = Arc::clone(®istry_client); + + move || { + Arc::new(CryptoComponent::new( + &crypto_config, // config + Some(tokio::runtime::Handle::current()), // tokio_runtime_handle + registry_client, // registry_client + no_op_logger(), // logger + None, // metrics + )) + } + }) + .await?; + + let pk = tokio::task::spawn_blocking({ + let c = Arc::clone(&c); + + move || { + c.current_node_public_keys() + .map_err(|err| anyhow!("failed to retrieve public key: {err:?}"))? + .node_signing_public_key + .context("missing node public key") + } + }) + .await??; + + let nid = derive_node_id(&pk).expect("failed to derive node id"); + + // Custom Signer + let s = Sender::Node { + pub_key: pk.key_value, + sign: Arc::new(move |msg: &MessageId| { + #[allow(clippy::disallowed_methods)] + let sig = tokio::task::block_in_place(|| { + c.sign_basic( + msg, // message + nid, // signer + registry_client.get_latest_version(), // registry_version + ) + .map(|value| value.get().0) + .map_err(|err| anyhow!("failed to sign message: {err:?}")) + })?; + + Ok(sig) + }), + }; + + // Agent + let (proto, port) = [ + // HTTP + cli.listen.listen_http_port.map(|p| ("http", p)), + // HTTPS + #[cfg(feature = "tls")] + cli.listen.listen_https_port.map(|p| ("https", p)), + ] + .into_iter() + // Choose first non-none option + .flatten() + .next() + .ok_or(anyhow!("missing http and https listen ports"))?; + + let ag = Agent::new( + format!("{proto}://localhost:{port}").parse()?, // url + s, // sender + ); + + // Canister + let c = AnonymizationCanister::new( + ag, // agent + log_anonymization_cid, // canister_id + ); + + // Rng + let rng = Box::new(OsRng); + + // Tracker + let tracker = AnonymizationTracker::new(rng, c.into())?; + + Some(tracker) + } + _ => None, + }; + TokioScope::scope_and_block(move |s| { if let Some(v) = registry_replicator { s.spawn(async move { @@ -358,6 +506,16 @@ pub async fn main(cli: Cli) -> Result<(), Error> { }); } + // Anonymization Tracker + if let Some(mut t) = tracker { + s.spawn(async move { + t.track(|value| { + anonymization_salt.store(Some(Arc::new(value))); + }) + .await + }); + } + // HTTP servers s.spawn(async move { metrics_server @@ -414,25 +572,13 @@ struct RegistrySetupResult( // Sets up registry-related stuff fn setup_registry( cli: &Cli, + local_store: Arc, + registry_client: Arc, registry_snapshot: Arc>, persister: WithMetricsPersist, http_client_check: Arc, metrics_registry: &Registry, ) -> Result { - // Registry Client - let local_store = Arc::new(LocalStoreImpl::new( - cli.registry.registry_local_store_path.clone().unwrap(), - )); - - let registry_client = Arc::new(RegistryClientImpl::new( - local_store.clone(), // data_provider - None, // metrics_registry - )); - - registry_client - .fetch_and_start_polling() - .context("failed to start registry client")?; - // Snapshots let (channel_snapshot_send, channel_snapshot_recv) = tokio::sync::watch::channel(None); let snapshot_runner = WithMetricsSnapshot( @@ -661,6 +807,7 @@ pub fn setup_router( cli: &Cli, metrics_registry: &Registry, cache: Option>, + anonymization_salt: Arc>>, ) -> Router { let proxy_router = ProxyRouter::new( http_client.clone(), @@ -735,6 +882,7 @@ pub fn setup_router( metrics_registry, "http_request", cli.obs.obs_log_failed_requests_only, + anonymization_salt, ), metrics::metrics_middleware, ), diff --git a/rs/boundary_node/ic_boundary/src/metrics.rs b/rs/boundary_node/ic_boundary/src/metrics.rs index 8a499aef57d..38ff6ad18ff 100644 --- a/rs/boundary_node/ic_boundary/src/metrics.rs +++ b/rs/boundary_node/ic_boundary/src/metrics.rs @@ -1,6 +1,10 @@ #![allow(clippy::disallowed_types)] -use std::{sync::Arc, time::Instant}; +use std::{ + hash::{DefaultHasher, Hash, Hasher}, + sync::Arc, + time::Instant, +}; use anyhow::Error; use arc_swap::ArcSwapOption; @@ -356,10 +360,16 @@ pub struct HttpMetricParams { pub durationer: HistogramVec, pub request_sizer: HistogramVec, pub response_sizer: HistogramVec, + pub anonymization_salt: Arc>>, } impl HttpMetricParams { - pub fn new(registry: &Registry, action: &str, log_failed_requests_only: bool) -> Self { + pub fn new( + registry: &Registry, + action: &str, + log_failed_requests_only: bool, + anonymization_salt: Arc>>, + ) -> Self { const LABELS_HTTP: &[&str] = &[ "request_type", "status_code", @@ -408,6 +418,8 @@ impl HttpMetricParams { registry ) .unwrap(), + + anonymization_salt, } } } @@ -492,6 +504,12 @@ pub async fn metrics_middleware( .map(|x| x.remote_addr.family()) .unwrap_or("0"); + let remote_addr = request + .extensions() + .get::>() + .map(|x| x.remote_addr.ip().to_canonical().to_string()) + .unwrap_or_default(); + let request_type = &request .extensions() .get::() @@ -561,6 +579,7 @@ pub async fn metrics_middleware( durationer, request_sizer, response_sizer, + anonymization_salt, } = metric_params; let (parts, body) = response.into_parts(); @@ -623,6 +642,25 @@ pub async fn metrics_middleware( .with_label_values(labels) .observe(response_size as f64); + // Anonymization + let s = anonymization_salt.load(); + + let hash_fn = |v: &str| -> String { + if s.is_none() { + return "N/A".to_string(); + } + + let mut h = DefaultHasher::new(); + v.hash(&mut h); + s.hash(&mut h); + + format!("{:x}", h.finish()) + }; + + let remote_addr = hash_fn(&remote_addr); + let sender = hash_fn(&sender.unwrap_or_default()); + + // Log if !log_failed_requests_only || failed { info!( action, @@ -638,6 +676,7 @@ pub async fn metrics_middleware( canister_id_actual = canister_id_actual.map(|x| x.to_string()), canister_id_cbor = ctx.canister_id.map(|x| x.to_string()), sender, + remote_addr, method = ctx.method_name, duration = proc_duration, duration_full = full_duration, diff --git a/rs/boundary_node/ic_boundary/src/test_utils.rs b/rs/boundary_node/ic_boundary/src/test_utils.rs index b87f72bb912..b984b56f710 100644 --- a/rs/boundary_node/ic_boundary/src/test_utils.rs +++ b/rs/boundary_node/ic_boundary/src/test_utils.rs @@ -315,6 +315,8 @@ pub fn setup_test_router( let subnets = registry_snapshot.load_full().unwrap().subnets.clone(); persister.persist(subnets.clone()); + let salt: Arc>> = Arc::new(ArcSwapOption::empty()); + let router = setup_router( registry_snapshot, routing_table, @@ -326,6 +328,7 @@ pub fn setup_test_router( enable_cache.then_some(Arc::new( Cache::new(10485760, 262144, Duration::from_secs(1), false).unwrap(), )), + salt, ); let router = router.layer(axum::middleware::from_fn(add_conninfo)); diff --git a/rs/canister_client/BUILD.bazel b/rs/canister_client/BUILD.bazel index 7f200a2c24a..85ddd2fce44 100644 --- a/rs/canister_client/BUILD.bazel +++ b/rs/canister_client/BUILD.bazel @@ -2,6 +2,8 @@ load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") # The package is deprecated, NET-1274. package(default_visibility = [ + "//rs/boundary_node/anonymization/client:__pkg__", + "//rs/boundary_node/ic_boundary:__pkg__", "//rs/cup_explorer:__pkg__", "//rs/http_endpoints/public:__pkg__", "//rs/nns/init:__pkg__", @@ -9,7 +11,6 @@ package(default_visibility = [ "//rs/registry/admin:__pkg__", "//rs/registry/nns_data_provider:__pkg__", "//rs/replay:__pkg__", - "//rs/rosetta-api:__subpackages__", "//rs/rust_canisters/canister_test:__pkg__", "//rs/tests:__subpackages__", "//rs/workload_generator:__pkg__", @@ -17,6 +18,7 @@ package(default_visibility = [ DEPENDENCIES = [ # Keep sorted. + "//rs/canister_client/read_state_response_parser", "//rs/canister_client/sender", "//rs/canonical_state", "//rs/certification", diff --git a/rs/canister_client/Cargo.toml b/rs/canister_client/Cargo.toml index dc12b41729c..c285522c6d0 100644 --- a/rs/canister_client/Cargo.toml +++ b/rs/canister_client/Cargo.toml @@ -20,6 +20,7 @@ ic-crypto-secp256k1 = { path = "../crypto/secp256k1" } ic-crypto-tree-hash = { path = "../crypto/tree_hash" } ic-management-canister-types = { path = "../types/management_canister_types" } ic-protobuf = { path = "../protobuf" } +ic-read-state-response-parser = { path = "./read_state_response_parser" } ic-types = { path = "../types/types" } itertools = { workspace = true } prost = { workspace = true } diff --git a/rs/canister_client/read_state_response_parser/BUILD.bazel b/rs/canister_client/read_state_response_parser/BUILD.bazel new file mode 100644 index 00000000000..70b27051fc2 --- /dev/null +++ b/rs/canister_client/read_state_response_parser/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") + +package(default_visibility = ["//visibility:public"]) + +DEPENDENCIES = [ + # Keep sorted. + "//rs/canonical_state", + "//rs/certification", + "//rs/crypto/tree_hash", + "//rs/tree_deserializer", + "//rs/types/types", + "@crate_index//:serde", + "@crate_index//:serde_cbor", +] + +DEV_DEPENDENCIES = [ + # Keep sorted. + "//rs/certification/test-utils", +] + +rust_library( + name = "read_state_response_parser", + srcs = glob(["src/**/*.rs"]), + crate_name = "ic_read_state_response_parser", + deps = DEPENDENCIES, +) + +rust_test( + name = "tests", + crate = ":read_state_response_parser", + deps = DEV_DEPENDENCIES, +) diff --git a/rs/canister_client/read_state_response_parser/Cargo.toml b/rs/canister_client/read_state_response_parser/Cargo.toml new file mode 100644 index 00000000000..d44e4b514d2 --- /dev/null +++ b/rs/canister_client/read_state_response_parser/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "ic-read-state-response-parser" +version.workspace = true +authors.workspace = true +edition.workspace = true +description.workspace = true +documentation.workspace = true + +[dependencies] +ic-canonical-state = { path = "../../canonical_state" } +ic-certification = { path = "../../certification" } +ic-crypto-tree-hash = { path = "../../crypto/tree_hash" } +ic-types = { path = "../../types/types" } +serde = { workspace = true } +serde_cbor = { workspace = true } +tree-deserializer = { path = "../../tree_deserializer" } + +[dev-dependencies] +ic-certification-test-utils = { path = "../../certification/test-utils" } + diff --git a/rs/canister_client/read_state_response_parser/src/lib.rs b/rs/canister_client/read_state_response_parser/src/lib.rs new file mode 100644 index 00000000000..f12c41ce6b0 --- /dev/null +++ b/rs/canister_client/read_state_response_parser/src/lib.rs @@ -0,0 +1,297 @@ +use ic_canonical_state::encoding::types::SubnetMetrics; +use ic_crypto_tree_hash::{LabeledTree, LookupStatus, MixedHashTree}; +use ic_types::{ + crypto::threshold_sig::ThresholdSigPublicKey, + messages::{HttpReadStateResponse, MessageId}, + CanisterId, SubnetId, +}; +use serde::Deserialize; +use serde_cbor::value::Value as CBOR; +use std::collections::BTreeMap; +use std::convert::TryFrom; + +// An auxiliary structure that mirrors the request statuses +// encoded in a certificate, starting from the root of the tree. +#[derive(Debug, Deserialize)] +struct RequestStatuses { + request_status: Option>, +} + +#[derive(Eq, PartialEq, Debug, Deserialize)] +pub struct RequestStatus { + pub status: String, + pub reply: Option>, + pub reject_message: Option, +} + +impl RequestStatus { + fn unknown() -> Self { + RequestStatus { + status: "unknown".to_string(), + reply: None, + reject_message: None, + } + } +} + +/// Given a CBOR response from a `read_state` and a `request_id` extracts +/// the `RequestStatus` if available. +pub fn parse_read_state_response( + request_id: &MessageId, + effective_canister_id: &CanisterId, + root_pk: Option<&ThresholdSigPublicKey>, + message: CBOR, +) -> Result { + let response = serde_cbor::value::from_value::(message) + .map_err(|source| format!("decoding to HttpReadStateResponse failed: {}", source))?; + + let certificate = match root_pk { + Some(pk) => { + ic_certification::verify_certificate(&response.certificate, effective_canister_id, pk) + .map_err(|source| format!("verifying certificate failed: {}", source))? + } + None => serde_cbor::from_slice(response.certificate.as_slice()) + .map_err(|source| format!("decoding Certificate failed: {}", source))?, + }; + + match certificate + .tree + .lookup(&[&b"request_status"[..], request_id.as_ref()]) + { + LookupStatus::Found(_) => (), + // TODO(MR-249): return an error in the Unknown case once the replica + // implements absence proofs. + LookupStatus::Absent | LookupStatus::Unknown => return Ok(RequestStatus::unknown()), + } + + // Parse the tree. + let tree = LabeledTree::try_from(certificate.tree) + .map_err(|e| format!("parsing tree in certificate failed: {:?}", e))?; + + let request_statuses = + RequestStatuses::deserialize(tree_deserializer::LabeledTreeDeserializer::new(&tree)) + .map_err(|err| format!("deserializing request statuses failed: {:?}", err))?; + + Ok(match request_statuses.request_status { + Some(mut request_status_map) => request_status_map + .remove(request_id) + .unwrap_or_else(RequestStatus::unknown), + None => RequestStatus::unknown(), + }) +} + +/// Given a CBOR response from a subnet `read_state` and a `subnet_id` extracts +/// the `SubnetMetrics` if available. +pub fn parse_subnet_read_state_response( + subnet_id: &SubnetId, + root_pk: Option<&ThresholdSigPublicKey>, + message: CBOR, +) -> Result { + let response = serde_cbor::value::from_value::(message) + .map_err(|source| format!("decoding to HttpReadStateResponse failed: {}", source))?; + + let certificate = match root_pk { + Some(pk) => ic_certification::verify_certificate_for_subnet_read_state( + &response.certificate, + subnet_id, + pk, + ) + .map_err(|source| format!("verifying certificate failed: {}", source))?, + None => serde_cbor::from_slice(response.certificate.as_slice()) + .map_err(|source| format!("decoding Certificate failed: {}", source))?, + }; + + let subnet_metrics_leaf = + match certificate + .tree + .lookup(&[&b"subnet"[..], subnet_id.get().as_ref(), &b"metrics"[..]]) + { + LookupStatus::Found(subnet_metrics_leaf) => subnet_metrics_leaf.clone(), + LookupStatus::Absent | LookupStatus::Unknown => return Ok(SubnetMetrics::default()), + }; + + match subnet_metrics_leaf { + MixedHashTree::Leaf(bytes) => { + let subnet_metrics: SubnetMetrics = serde_cbor::from_slice(&bytes) + .map_err(|err| format!("deserializing subnet_metrics failed: {:?}", err))?; + Ok(subnet_metrics) + } + tree => Err(format!("Expected subnet metrics leaf but found {:?}", tree)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ic_certification_test_utils::CertificateBuilder; + use ic_certification_test_utils::CertificateData; + use ic_crypto_tree_hash::Digest; + use ic_crypto_tree_hash::Label; + use ic_types::messages::Blob; + use ic_types::CanisterId; + use serde::Serialize; + + fn to_self_describing_cbor(e: &T) -> serde_cbor::Result> { + let mut serialized_bytes = Vec::new(); + let mut serializer = serde_cbor::Serializer::new(&mut serialized_bytes); + serializer.self_describe()?; + e.serialize(&mut serializer)?; + Ok(serialized_bytes) + } + + #[test] + fn test_parse_read_state_response_unknown() { + let labeled_tree = LabeledTree::try_from(MixedHashTree::Labeled( + "time".into(), + Box::new(MixedHashTree::Leaf(vec![1])), + )) + .unwrap(); + let data = CertificateData::CustomTree(labeled_tree); + let (certificate, root_pk, _) = CertificateBuilder::new(data).build(); + + let certificate_cbor: Vec = to_self_describing_cbor(&certificate).unwrap(); + + let response = HttpReadStateResponse { + certificate: Blob(certificate_cbor), + }; + + let response_cbor: Vec = to_self_describing_cbor(&response).unwrap(); + + let response: CBOR = serde_cbor::from_slice(response_cbor.as_slice()).unwrap(); + + let request_id: MessageId = MessageId::from([0; 32]); + assert_eq!( + parse_read_state_response(&request_id, &CanisterId::from(1), Some(&root_pk), response), + Ok(RequestStatus::unknown()) + ); + } + + #[test] + fn test_parse_read_state_response_replied() { + let tree = MixedHashTree::Fork(Box::new(( + MixedHashTree::Labeled( + "request_status".into(), + Box::new(MixedHashTree::Labeled( + vec![ + 184, 255, 145, 192, 128, 156, 132, 76, 67, 213, 87, 237, 189, 136, 206, + 184, 254, 192, 233, 210, 142, 173, 27, 123, 112, 187, 82, 222, 130, 129, + 245, 41, + ] + .into(), + Box::new(MixedHashTree::Fork(Box::new(( + MixedHashTree::Labeled( + "reply".into(), + Box::new(MixedHashTree::Leaf(vec![68, 73, 68, 76, 0, 0])), + ), + MixedHashTree::Labeled( + "status".into(), + Box::new(MixedHashTree::Leaf(b"replied".to_vec())), + ), + )))), + )), + ), + MixedHashTree::Labeled("time".into(), Box::new(MixedHashTree::Leaf(vec![1]))), + ))); + let labeled_tree = LabeledTree::try_from(tree).unwrap(); + let data = CertificateData::CustomTree(labeled_tree); + let (certificate, root_pk, _) = CertificateBuilder::new(data).build(); + + let certificate_cbor: Vec = to_self_describing_cbor(&certificate).unwrap(); + + let response = HttpReadStateResponse { + certificate: Blob(certificate_cbor), + }; + + let response_cbor: Vec = to_self_describing_cbor(&response).unwrap(); + + let response: CBOR = serde_cbor::from_slice(response_cbor.as_slice()).unwrap(); + + // Request ID that exists. + let request_id: MessageId = MessageId::from([ + 184, 255, 145, 192, 128, 156, 132, 76, 67, 213, 87, 237, 189, 136, 206, 184, 254, 192, + 233, 210, 142, 173, 27, 123, 112, 187, 82, 222, 130, 129, 245, 41, + ]); + + assert_eq!( + parse_read_state_response( + &request_id, + &CanisterId::from(1), + Some(&root_pk), + response.clone() + ), + Ok(RequestStatus { + status: "replied".to_string(), + reply: Some(vec![68, 73, 68, 76, 0, 0]), + reject_message: None + }), + ); + + // Request ID that doesn't exist. + let request_id: MessageId = MessageId::from([0; 32]); + assert_eq!( + parse_read_state_response(&request_id, &CanisterId::from(1), Some(&root_pk), response), + Ok(RequestStatus::unknown()) + ); + } + + #[test] + fn test_parse_read_state_response_pruned() { + fn mklabeled(l: impl Into