diff --git a/.github/workflows/build-push-containers-all.yml b/.github/workflows/build-push-containers-all.yml index b29e0e20e..a93f1e3a2 100644 --- a/.github/workflows/build-push-containers-all.yml +++ b/.github/workflows/build-push-containers-all.yml @@ -7,14 +7,29 @@ on: - labeled - opened - synchronize + push: branches: - - '**' + - main tags: - '[0-9]*.[0-9]*.[0-9]*' jobs: + + read-github-context-object: + runs-on: ubuntu-latest + steps: + # I know this is not pretty (debug information) but it's useful to have this + - name: Read github context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: | + echo "GITHUB_SHA=${GITHUB_SHA}" + echo "INFO: github context object content below" + echo "${GITHUB_CONTEXT}" + build-push-checked-containers: + if: contains(github.event.pull_request.labels.*.name, 'cicd:movement-containers') || github.ref == 'refs/heads/main' || github.event.label.name == 'cicd:movement-containers' uses: ./.github/workflows/build-push-container.yml name: Build Push Checked Containers secrets: inherit @@ -36,6 +51,7 @@ jobs: container_name: ${{ matrix.container_name }} build-push-checked-manifest: + if: contains(github.event.pull_request.labels.*.name, 'cicd:movement-containers') || github.ref == 'refs/heads/main' || github.event.label.name == 'cicd:movement-containers' uses: ./.github/workflows/build-push-manifest.yml name: Build Push Checked Manifest needs: @@ -60,7 +76,7 @@ jobs: container-checks: if: contains(github.event.pull_request.labels.*.name, 'cicd:movement-containers') || github.ref == 'refs/heads/main' || github.event.label.name == 'cicd:movement-containers' - runs-on: buildjet-8vcpu-ubuntu-2204 + runs-on: buildjet-16vcpu-ubuntu-2204 needs: - build-push-checked-manifest steps: @@ -110,6 +126,8 @@ jobs: ### Unchecked containers build-push-unchecked-containers: + if: contains(github.event.pull_request.labels.*.name, 'cicd:bridge-containers') || github.ref == 'refs/heads/main' || github.event.label.name == 'cicd:bridge-containers' + uses: ./.github/workflows/build-push-container.yml name: Build Push Unchecked Containers secrets: inherit diff --git a/.github/workflows/checks-all.yml b/.github/workflows/checks-all.yml index 14b8f9bd9..21f2acb63 100755 --- a/.github/workflows/checks-all.yml +++ b/.github/workflows/checks-all.yml @@ -8,24 +8,39 @@ on: - synchronize push: branches: - - '**' + - main + tags: + - '[0-9]*.[0-9]*.[0-9]*' jobs: - + + read-github-context-object: + runs-on: ubuntu-latest + steps: + # I know this is not pretty (debug information) but it's useful to have this + - name: Read github context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: | + echo "GITHUB_SHA=${GITHUB_SHA}" + echo "INFO: github context object content below" + echo "${GITHUB_CONTEXT}" + build: strategy: matrix: include: - os: ubuntu-22.04 arch: x86_64 - runs-on: buildjet-8vcpu-ubuntu-2204 - - os: macos-13-latest + runs-on: buildjet-16vcpu-ubuntu-2204 + - os: macos-15 arch: arm64 - runs-on: macos-13-xlarge + runs-on: macos-15 runs-on: ${{ matrix.runs-on }} steps: + - name: Checkout repository uses: actions/checkout@v4 @@ -42,10 +57,10 @@ jobs: include: - os: ubuntu-22.04 arch: x86_64 - runs-on: buildjet-8vcpu-ubuntu-2204 - - os: macos-13-latest + runs-on: buildjet-16vcpu-ubuntu-2204 + - os: macos-15 arch: arm64 - runs-on: macos-13-xlarge + runs-on: macos-15 runs-on: ${{ matrix.runs-on }} @@ -183,7 +198,7 @@ jobs: include: - os: ubuntu-22.04 arch: x86_64 - runs-on: buildjet-8vcpu-ubuntu-2204 + runs-on: buildjet-16vcpu-ubuntu-2204 runs-on: ${{ matrix.runs-on }} @@ -239,7 +254,7 @@ jobs: include: - os: ubuntu-22.04 arch: x86_64 - runs-on: buildjet-8vcpu-ubuntu-2204 + runs-on: buildjet-16vcpu-ubuntu-2204 runs-on: ${{ matrix.runs-on }} @@ -317,7 +332,7 @@ jobs: include: - os: ubuntu-22.04 arch: x86_64 - runs-on: buildjet-8vcpu-ubuntu-2204 + runs-on: buildjet-16vcpu-ubuntu-2204 runs-on: ${{ matrix.runs-on }} @@ -372,7 +387,7 @@ jobs: include: - os: ubuntu-22.04 arch: x86_64 - runs-on: buildjet-8vcpu-ubuntu-2204 + runs-on: buildjet-16vcpu-ubuntu-2204 runs-on: ${{ matrix.runs-on }} @@ -444,7 +459,7 @@ jobs: # include: # - os: ubuntu-22.04 # arch: x86_64 -# runs-on: buildjet-8vcpu-ubuntu-2204 +# runs-on: buildjet-16vcpu-ubuntu-2204 # # runs-on: ${{ matrix.runs-on }} # diff --git a/.github/workflows/hsm-demo-containers.yml b/.github/workflows/hsm-demo-containers.yml deleted file mode 100644 index 95a3201a0..000000000 --- a/.github/workflows/hsm-demo-containers.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Build Push Containers - -on: - pull_request: - types: - - labeled - - opened - - synchronize - push: - branches: - - '**' - -jobs: - - hsm-demo-build: - if: github.event.label.name == 'cicd:hsm-demo-containers' || github.ref == 'refs/heads/main' - permissions: - contents: read - packages: write - strategy: - matrix: - architecture: [x86_64, arm64] - - runs-on: ${{ matrix.architecture == 'x86_64' && 'buildjet-8vcpu-ubuntu-2204' || 'buildjet-8vcpu-ubuntu-2204-arm' }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: true - ref: ${{ github.event.pull_request.head.ref || github.ref }} - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Login to Docker Hub to Avoid Rate Limiting - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - name: Build and Push Docker image movement - run: | - ./scripts/movement/build-push-image hsm-demo - - hsm-demo-manifest: - permissions: - contents: read - packages: write - needs: hsm-demo-build - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: true - ref: ${{ github.event.pull_request.head.ref || github.ref }} - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Login to Docker Hub to Avoid Rate Limiting - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - name: Build and Push Docker image movement - run: | - ./scripts/movement/manifest hsm-demo \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 38a44c855..64d8547a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5041,9 +5041,6 @@ name = "beef" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" -dependencies = [ - "serde", -] [[package]] name = "bellpepper" @@ -5289,9 +5286,9 @@ dependencies = [ [[package]] name = "blockstore" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7679095248a6dc7555fae81154ed1baef264383c16621ef881a219576c72a9be" +checksum = "0a8962daed8fb337472d9c4215006443acba1e40c6c91c9d4a3f440d1fb30436" dependencies = [ "cid", "dashmap 6.1.0", @@ -5859,91 +5856,45 @@ dependencies = [ [[package]] name = "celestia-proto" -version = "0.5.0" -source = "git+https://github.com/movementlabsxyz/lumina?rev=2d16e6733949f6bf70849eb60c14114e6d8ea63e#2d16e6733949f6bf70849eb60c14114e6d8ea63e" +version = "0.7.0" +source = "git+https://github.com/eigerco/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" dependencies = [ - "celestia-tendermint-proto", + "bytes 1.8.0", "prost 0.13.3", - "prost-build 0.12.6", + "prost-build", "prost-types 0.13.3", "protox", "serde", + "subtle-encoding", + "tendermint-proto", ] [[package]] name = "celestia-rpc" -version = "0.7.1" -source = "git+https://github.com/movementlabsxyz/lumina?rev=2d16e6733949f6bf70849eb60c14114e6d8ea63e#2d16e6733949f6bf70849eb60c14114e6d8ea63e" +version = "0.9.0" +source = "git+https://github.com/eigerco/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" dependencies = [ "async-trait", "celestia-types", - "futures", "http 1.1.0", - "jsonrpsee 0.24.7", + "jsonrpsee", + "prost 0.13.3", "serde", "thiserror 1.0.69", "tracing", ] -[[package]] -name = "celestia-tendermint" -version = "0.32.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce8c92a01145f79a0f3ac7c44a43a9b5ee58e8a4c716b56d98833a3848db1afd" -dependencies = [ - "bytes 1.8.0", - "celestia-tendermint-proto", - "digest 0.10.7", - "ed25519 2.2.3", - "ed25519-consensus", - "flex-error", - "futures", - "num-traits", - "once_cell", - "prost 0.12.6", - "prost-types 0.12.6", - "serde", - "serde_bytes", - "serde_json", - "serde_repr", - "sha2 0.10.8", - "signature 2.2.0", - "subtle", - "subtle-encoding", - "time", - "zeroize", -] - -[[package]] -name = "celestia-tendermint-proto" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bbd5c2345d29ea85566ad8a1114cc0dfb20dacf8f9e0722d64886d0e1e064f7" -dependencies = [ - "bytes 1.8.0", - "flex-error", - "num-derive", - "num-traits", - "prost 0.13.3", - "prost-types 0.13.3", - "serde", - "serde_bytes", - "subtle-encoding", - "time", -] - [[package]] name = "celestia-types" -version = "0.8.0" -source = "git+https://github.com/movementlabsxyz/lumina?rev=2d16e6733949f6bf70849eb60c14114e6d8ea63e#2d16e6733949f6bf70849eb60c14114e6d8ea63e" +version = "0.10.0" +source = "git+https://github.com/eigerco/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" dependencies = [ "base64 0.22.1", "bech32", + "bitvec 1.0.1", "blockstore", "bytes 1.8.0", "celestia-proto", - "celestia-tendermint", - "celestia-tendermint-proto", "cid", "const_format", "enum_dispatch", @@ -5957,6 +5908,8 @@ dependencies = [ "serde", "serde_repr", "sha2 0.10.8", + "tendermint", + "tendermint-proto", "thiserror 1.0.69", "time", ] @@ -6271,6 +6224,7 @@ version = "0.0.2" dependencies = [ "anyhow", "futures", + "itertools 0.13.0", "tokio", "tracing", ] @@ -7723,16 +7677,6 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" -[[package]] -name = "eyre" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" -dependencies = [ - "indenter", - "once_cell", -] - [[package]] name = "fail" version = "0.5.1" @@ -7937,7 +7881,6 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c606d892c9de11507fa0dcffc116434f94e105d0bbdc4e405b61519464c49d7b" dependencies = [ - "eyre", "paste", ] @@ -8931,7 +8874,7 @@ dependencies = [ "aws-config", "aws-sdk-kms", "axum 0.6.20", - "base64 0.13.1", + "base64 0.22.1", "clap 4.5.21", "dotenv", "ed25519 2.2.3", @@ -9500,12 +9443,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "indenter" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" - [[package]] name = "indexmap" version = "1.9.3" @@ -9766,15 +9703,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jsonrpsee" -version = "0.20.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138572befc78a9793240645926f30161f8b4143d2be18d09e44ed9814bd7ee2c" -dependencies = [ - "jsonrpsee-types 0.20.4", -] - [[package]] name = "jsonrpsee" version = "0.24.7" @@ -9784,7 +9712,7 @@ dependencies = [ "jsonrpsee-core", "jsonrpsee-http-client", "jsonrpsee-proc-macros", - "jsonrpsee-types 0.24.7", + "jsonrpsee-types", "jsonrpsee-ws-client", "tracing", ] @@ -9825,7 +9753,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "http-body-util", - "jsonrpsee-types 0.24.7", + "jsonrpsee-types", "pin-project 1.1.7", "rustc-hash 2.0.0", "serde", @@ -9849,7 +9777,7 @@ dependencies = [ "hyper-rustls 0.27.3", "hyper-util", "jsonrpsee-core", - "jsonrpsee-types 0.24.7", + "jsonrpsee-types", "rustls 0.23.17", "rustls-platform-verifier", "serde", @@ -9874,20 +9802,6 @@ dependencies = [ "syn 2.0.87", ] -[[package]] -name = "jsonrpsee-types" -version = "0.20.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3264e339143fe37ed081953842ee67bfafa99e3b91559bdded6e4abd8fc8535e" -dependencies = [ - "anyhow", - "beef", - "serde", - "serde_json", - "thiserror 1.0.69", - "tracing", -] - [[package]] name = "jsonrpsee-types" version = "0.24.7" @@ -9909,7 +9823,7 @@ dependencies = [ "http 1.1.0", "jsonrpsee-client-transport", "jsonrpsee-core", - "jsonrpsee-types 0.24.7", + "jsonrpsee-types", "url", ] @@ -10605,6 +10519,7 @@ dependencies = [ "maptos-framework-release-util", "movement-collections", "movement-rest", + "movement-signer", "movement-signer-loader", "movement-types", "poem", @@ -11722,7 +11637,7 @@ name = "movement-algs" version = "0.0.2" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.13.0", "movement-types", "tokio", ] @@ -11809,7 +11724,7 @@ dependencies = [ "dot-movement", "ed25519-dalek 2.1.1", "futures", - "itertools 0.12.1", + "itertools 0.13.0", "maptos-execution-util", "mcr-settlement-client", "movement-config", @@ -12052,6 +11967,7 @@ dependencies = [ "aptos-account-whitelist", "aptos-types", "async-stream", + "base64 0.22.1", "bcs 0.1.6 (git+https://github.com/movementlabsxyz/bcs.git?rev=bc16d2d39cabafaabd76173dd1b04b2aa170cf0c)", "celestia-rpc", "celestia-types", @@ -12060,7 +11976,6 @@ dependencies = [ "ecdsa 0.16.9", "godfig", "hex", - "jsonrpsee 0.20.4", "k256", "memseq-util", "movement-da-light-node-proto", @@ -12105,8 +12020,11 @@ dependencies = [ name = "movement-full-node" version = "0.0.2" dependencies = [ + "alloy-signer", "anyhow", + "aptos-crypto", "aptos-framework-elsa-to-biarritz-rc1-migration", + "aptos-sdk", "bcs 0.1.6 (git+https://github.com/movementlabsxyz/bcs.git?rev=bc16d2d39cabafaabd76173dd1b04b2aa170cf0c)", "chrono", "clap 4.5.21", @@ -12116,6 +12034,7 @@ dependencies = [ "godfig", "hex", "hyper 1.5.0", + "k256", "maptos-dof-execution", "mcr-settlement-client", "mcr-settlement-config", @@ -12246,7 +12165,7 @@ version = "0.0.2" dependencies = [ "anyhow", "async-trait", - "base64 0.13.1", + "base64 0.22.1", "dotenv", "movement-signer", "tokio", @@ -12305,6 +12224,7 @@ dependencies = [ "async-trait", "chrono", "ed25519-dalek 2.1.1", + "hex", "maptos-dof-execution", "maptos-execution-util", "movement-signer", @@ -14190,27 +14110,6 @@ dependencies = [ "prost-derive 0.13.3", ] -[[package]] -name = "prost-build" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" -dependencies = [ - "bytes 1.8.0", - "heck 0.5.0", - "itertools 0.12.1", - "log", - "multimap", - "once_cell", - "petgraph 0.6.5", - "prettyplease", - "prost 0.12.6", - "prost-types 0.12.6", - "regex", - "syn 2.0.87", - "tempfile", -] - [[package]] name = "prost-build" version = "0.13.3" @@ -14273,15 +14172,15 @@ dependencies = [ [[package]] name = "prost-reflect" -version = "0.13.1" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f5eec97d5d34bdd17ad2db2219aabf46b054c6c41bd5529767c9ce55be5898f" +checksum = "7b5edd582b62f5cde844716e66d92565d7faf7ab1445c8cebce6e00fba83ddb2" dependencies = [ "logos", "miette", "once_cell", - "prost 0.12.6", - "prost-types 0.12.6", + "prost 0.13.3", + "prost-types 0.13.3", ] [[package]] @@ -14338,28 +14237,28 @@ dependencies = [ [[package]] name = "protox" -version = "0.6.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac532509cee918d40f38c3e12f8ef9230f215f017d54de7dd975015538a42ce7" +checksum = "6f352af331bf637b8ecc720f7c87bf903d2571fa2e14a66e9b2558846864b54a" dependencies = [ "bytes 1.8.0", "miette", - "prost 0.12.6", + "prost 0.13.3", "prost-reflect", - "prost-types 0.12.6", + "prost-types 0.13.3", "protox-parse", "thiserror 1.0.69", ] [[package]] name = "protox-parse" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6c33f43516fe397e2f930779d720ca12cd057f7da4cd6326a0ef78d69dee96" +checksum = "a3a462d115462c080ae000c29a47f0b3985737e5d3a995fcdbcaa5c782068dde" dependencies = [ "logos", "miette", - "prost-types 0.12.6", + "prost-types 0.13.3", "thiserror 1.0.69", ] @@ -16789,6 +16688,51 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "tendermint" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9703e34d940c2a293804752555107f8dbe2b84ec4c6dd5203831235868105d2" +dependencies = [ + "bytes 1.8.0", + "digest 0.10.7", + "ed25519 2.2.3", + "ed25519-consensus", + "flex-error", + "futures", + "k256", + "num-traits", + "once_cell", + "prost 0.13.3", + "ripemd", + "serde", + "serde_bytes", + "serde_json", + "serde_repr", + "sha2 0.10.8", + "signature 2.2.0", + "subtle", + "subtle-encoding", + "tendermint-proto", + "time", + "zeroize", +] + +[[package]] +name = "tendermint-proto" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9e1705aa0fa5ecb2c6aa7fb78c2313c4a31158ea5f02048bf318f849352eb" +dependencies = [ + "bytes 1.8.0", + "flex-error", + "prost 0.13.3", + "serde", + "serde_bytes", + "subtle-encoding", + "time", +] + [[package]] name = "tera" version = "1.20.0" @@ -17372,7 +17316,7 @@ checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" dependencies = [ "prettyplease", "proc-macro2", - "prost-build 0.13.3", + "prost-build", "prost-types 0.13.3", "quote", "syn 2.0.87", diff --git a/Cargo.toml b/Cargo.toml index 0d0b79119..8b4eadcc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,7 +152,6 @@ movement-signing-eth = { path = "util/signing/integrations/eth" } vaultrs = { version = "0.7.3" } aws-sdk-kms = "1.51.0" google-cloud-kms = "0.6.0" -base64 = { version = "0.13.0" } # Serialization and Deserialization serde = "1.0" @@ -229,8 +228,8 @@ secp256k1 = { version = "0.27", default-features = false, features = [ ] } ## Celestia Dependencies -celestia-rpc = { git = "https://github.com/movementlabsxyz/lumina", rev = "2d16e6733949f6bf70849eb60c14114e6d8ea63e" } #{ version = "0.7.0" } -celestia-types = { git = "https://github.com/movementlabsxyz/lumina", rev = "2d16e6733949f6bf70849eb60c14114e6d8ea63e" } #{ version = "0.7.0" } +celestia-rpc = { git = "https://github.com/eigerco/lumina", rev = "c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5" } #{ version = "0.7.0" } +celestia-types = { git = "https://github.com/eigerco/lumina", rev = "c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5" } #{ version = "0.7.0" } # External Dependencies @@ -267,6 +266,7 @@ async-stream = "0.3.0" async-trait = "0.1.71" async-recursion = "1.1.1" auto_impl = "1.2.0" +base64 = { version = "0.22.1" } blake3 = { version = "1.4.0", features = ["traits-preview"] } bytes = { version = "1.2.1", default-features = false } chrono = "0.4.37" @@ -288,9 +288,8 @@ hex = { version = "0.4.3", default-features = false, features = [ ics23 = { version = "0.11.0" } k256 = { version = "0.13.3" } keccak-hash = "0.10.0" -itertools = { version = "0.12.1", default-features = false } +itertools = { version = "0.13.0", default-features = false } jmt = "0.9.0" -jsonrpsee = { version = "0.20.1", features = ["jsonrpsee-types"] } log = "0.4.21" mirai-annotations = "1.10.1" move-vm-integration-test-helpers = { path = "test-helpers/move-vm-integration-test-helpers" } diff --git a/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml b/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml new file mode 100644 index 000000000..14b526d2e --- /dev/null +++ b/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml @@ -0,0 +1,51 @@ +services: + setup: + image: ghcr.io/movementlabsxyz/movement-full-node-setup:${CONTAINER_REV} + container_name: setup + environment: + CELESTIA_NAMESPACE: "0xa21de06de7aa38c2a812" + CELESTIA_NETWORK: mainnet + # ws://celestia-light-node:26658 + CELESTIA_WEBSOCKET_CONNECTION_PROTOCOL: ws + CELESTIA_WEBSOCKET_CONNECTION_HOSTNAME: celestia-light-node + CELESTIA_WEBSOCKET_CONNECTION_PORT: 26658 + INDEXER_PROCESSOR_POSTGRES_CONNECTION_STRING: postgres://postgres:password@postgres:5432/postgres + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_REGION: ${AWS_DEFAULT_REGION} + volumes: + - ${DOT_MOVEMENT_PATH}:/.movement + healthcheck: + test: [ "CMD-SHELL", "echo 'health check'" ] + retries: 30 + interval: 10s + timeout: 5s + + celestia-light-node: + image: ghcr.io/celestiaorg/celestia-node:v0.21.5 + container_name: celestia-light-node + command: | + celestia light start + --core.ip consensus.celestia.mainnet.movementinfra.xyz + --core.port 9090 + --rpc.addr 0.0.0.0 + --p2p.network celestia + --node.store /.movement/celestia/movement/.celestia-light + --keyring.backend test + --keyring.keyname movement_celestia_light + --log.level WARN + environment: + - NODE_TYPE=light + - P2P_NETWORK=celestia + - NODE_STORE=/.movement/celestia/movement/.celestia-light + user: root:root + volumes: + - ${DOT_MOVEMENT_PATH}/celestia:/.movement/celestia + ports: + - 26658:26658 + depends_on: + setup: + condition: service_healthy + healthcheck: + test: "nc -zv 0.0.0.0 26658" + restart: on-failure:3 diff --git a/docker/compose/movement-full-node/docker-compose.mainnet-leader.yml b/docker/compose/movement-full-node/docker-compose.mainnet-leader.yml new file mode 100644 index 000000000..dd273568c --- /dev/null +++ b/docker/compose/movement-full-node/docker-compose.mainnet-leader.yml @@ -0,0 +1,11 @@ +services: + + # turn off movement-faucet-service + movement-faucet-service: + image: busybox + container_name: movement-faucet-service + command: sleep infinity + healthcheck: + test: [ "CMD-SHELL", "echo 'health check'" ] + retries: 3 + start_period: 3s \ No newline at end of file diff --git a/networks/movement/indexer/src/main.rs b/networks/movement/indexer/src/main.rs index e5fdd9391..d59fea36e 100644 --- a/networks/movement/indexer/src/main.rs +++ b/networks/movement/indexer/src/main.rs @@ -58,6 +58,15 @@ fn main() -> Result<(), anyhow::Error> { None }; + // ANS processor. + let ans_indexer_config = build_processor_conf( + "ans_processor + ans_v1_primary_names_table_handle: temp + ans_v1_name_records_table_handle: temp + ans_v2_contract_address: 0x67bf15b3eed0fc62deea9630bbbd1d48842550655140f913699a1ca7e6f727d8", + &maptos_config, + )?; + let num_cpus = num_cpus::get(); let worker_threads = (num_cpus * RUNTIME_WORKER_MULTIPLIER).max(16); println!( @@ -94,6 +103,7 @@ fn main() -> Result<(), anyhow::Error> { set.spawn(async move { token_indexer_config.run().await }); set.spawn(async move { tokenv2_indexer_config.run().await }); } + set.spawn(async move { ans_indexer_config.run().await }); while let Some(res) = set.join_next().await { tracing::error!("An Error occurs during indexer execution: {res:?}"); @@ -122,14 +132,8 @@ fn build_processor_conf( .map(|t| t.parse().unwrap_or(10)) .unwrap_or(10); - // If the starting version is not defined, don't put a default value in the conf. - let starting_version_entry = std::env::var("INDEXER_STARTING_VERSION") - .map(|t| t.parse().unwrap_or(0)) - .map(|t| format!("starting_version: {}", t)) - .unwrap_or(String::new()); - //create config file - let indexer_config_content = format!( + let mut indexer_config_content = format!( "processor_config: type: {} postgres_connection_string: {} @@ -137,8 +141,7 @@ indexer_grpc_data_service_address: {} indexer_grpc_http2_ping_interval_in_secs: {} indexer_grpc_http2_ping_timeout_in_secs: {} auth_token: \"{}\" -default_sleep_time_between_request: {} -{}", +default_sleep_time_between_request: {}", processor_name, maptos_config.indexer_processor.postgres_connection_string, indexer_grpc_data_service_address, @@ -146,9 +149,16 @@ default_sleep_time_between_request: {} maptos_config.indexer.maptos_indexer_grpc_inactivity_ping_interval, maptos_config.indexer_processor.indexer_processor_auth_token, default_sleep_time_between_request, - starting_version_entry, ); + // If the starting version is not defined, don't put a default value in the conf. + if let Ok(start_version) = std::env::var("INDEXER_STARTING_VERSION") { + if let Ok(start_version) = start_version.parse::() { + indexer_config_content.push('\n'); + indexer_config_content.push_str(&format!("starting_version: {}", start_version)); + } + } + //let indexer_config_path = dot_movement.get_path().join("indexer_config.yaml"); let mut output_file = tempfile::NamedTempFile::new()?; write!(output_file, "{}", indexer_config_content)?; diff --git a/networks/movement/movement-config/src/migrations/elsa_to_biarritz_rc1/mod.rs b/networks/movement/movement-config/src/migrations/elsa_to_biarritz_rc1/mod.rs index a8be7a85a..a52eeaefa 100644 --- a/networks/movement/movement-config/src/migrations/elsa_to_biarritz_rc1/mod.rs +++ b/networks/movement/movement-config/src/migrations/elsa_to_biarritz_rc1/mod.rs @@ -607,8 +607,11 @@ mod tests { }) ); - let da_signers = match &config.celestia_da_light_node.celestia_da_light_node_config { - movement_da_util::config::Config::Local(local) => &local.da_light_node.da_signers, + let da_light_node_config = &config.celestia_da_light_node.celestia_da_light_node_config; + let da_signers = match &da_light_node_config.network { + movement_da_util::config::Network::Local => { + &da_light_node_config.da_light_node.da_signers + } _ => panic!("Expected Local"), }; diff --git a/networks/movement/movement-config/src/releases/biarritz_rc1/da_db.rs b/networks/movement/movement-config/src/releases/biarritz_rc1/da_db.rs index 07267a8c6..e155a9508 100644 --- a/networks/movement/movement-config/src/releases/biarritz_rc1/da_db.rs +++ b/networks/movement/movement-config/src/releases/biarritz_rc1/da_db.rs @@ -5,12 +5,15 @@ use serde::{Deserialize, Serialize}; pub struct Config { #[serde(default = "default_da_db_path")] pub da_db_path: String, + #[serde(default = "default_start_sync_height")] + pub start_sync_height: u64, } impl Default for Config { fn default() -> Self { - Self { da_db_path: default_da_db_path() } + Self { da_db_path: default_da_db_path(), start_sync_height: default_start_sync_height() } } } env_default!(default_da_db_path, "SUZUKA_DA_DB_PATH", String, "movement-da-db".to_string()); +env_default!(default_start_sync_height, "MOVEMENT_START_SYNC_HEIGHT", u64, 0); diff --git a/networks/movement/movement-full-node/Cargo.toml b/networks/movement/movement-full-node/Cargo.toml index 2a1a69330..0ec19059f 100644 --- a/networks/movement/movement-full-node/Cargo.toml +++ b/networks/movement/movement-full-node/Cargo.toml @@ -47,6 +47,10 @@ movement-signer = { workspace = true } movement-signer-loader = { workspace = true } syncador = { workspace = true } syncup = { workspace = true } +aptos-crypto = { workspace = true } +aptos-sdk = { workspace = true } +k256 = { workspace = true } +alloy-signer = { workspace = true } chrono = { workspace = true } [features] diff --git a/networks/movement/movement-full-node/src/admin/mod.rs b/networks/movement/movement-full-node/src/admin/mod.rs index a14fd74e5..590457527 100644 --- a/networks/movement/movement-full-node/src/admin/mod.rs +++ b/networks/movement/movement-full-node/src/admin/mod.rs @@ -5,6 +5,8 @@ pub mod governed_gas_pool; pub mod mcr; pub mod ops; pub mod rotate_key; +pub mod testkey; + use clap::Subcommand; #[derive(Subcommand, Debug)] @@ -24,6 +26,8 @@ pub enum Admin { Framework(framework::Framework), #[clap(subcommand)] Config(config::Config), + #[clap(subcommand)] + TestKey(testkey::TestKey), } impl Admin { @@ -36,6 +40,7 @@ impl Admin { Admin::Ops(ops) => ops.execute().await, Admin::Framework(framework) => framework.execute().await, Admin::Config(config) => config.execute().await, + Admin::TestKey(key) => key.execute().await, } } } diff --git a/networks/movement/movement-full-node/src/admin/testkey/mod.rs b/networks/movement/movement-full-node/src/admin/testkey/mod.rs new file mode 100644 index 000000000..2988b2e99 --- /dev/null +++ b/networks/movement/movement-full-node/src/admin/testkey/mod.rs @@ -0,0 +1,77 @@ +use aptos_crypto::ed25519::Ed25519PublicKey; +use aptos_sdk::types::transaction::authenticator::AuthenticationKey; +use clap::Parser; +use clap::Subcommand; +use k256::ecdsa::VerifyingKey; +use movement_signer::cryptography::ed25519::Ed25519; +use movement_signer::cryptography::secp256k1::Secp256k1; +use movement_signer::key::TryFromCanonicalString; +use movement_signer::Signing; +use movement_signer::Verify; +use movement_signer_loader::identifiers::SignerIdentifier; +use movement_signer_loader::{Load, LoadedSigner}; + +#[derive(Subcommand, Debug)] +#[clap(rename_all = "kebab-case", about = "Commands to test key name")] +pub enum TestKey { + Ed25519(TestKeyParam), + Secp256k1(TestKeyParam), +} + +impl TestKey { + pub async fn execute(&self) -> Result<(), anyhow::Error> { + match self { + TestKey::Ed25519(param) => param.execute_ed25519().await, + TestKey::Secp256k1(param) => param.execute_secp256k1().await, + } + } +} + +#[derive(Debug, Parser, Clone)] +#[clap(rename_all = "kebab-case", about = "Key to test.")] +pub struct TestKeyParam { + #[clap(default_value = "{maptos,maptos-storage,movement-da-db}/**", value_name = "DB PATTERN")] + pub name: String, +} + +impl TestKeyParam { + pub async fn execute_ed25519(&self) -> Result<(), anyhow::Error> { + let signer_identifier = SignerIdentifier::try_from_canonical_string(&self.name) + .map_err(|err| anyhow::anyhow!(err))?; + let loader: LoadedSigner = signer_identifier.load().await?; + + let public_key = Ed25519PublicKey::try_from(loader.public_key().await?.as_bytes())?; + let account_address = AuthenticationKey::ed25519(&public_key).account_address(); + + tracing::info!("Key loaded, account address:{account_address}"); + tracing::info!("Try to sign a message ..."); + + let message = b"Hello, world!"; + let signature = loader.sign(message).await?; + assert!(Ed25519::verify(message, &signature, &loader.public_key().await?)?); + + tracing::info!("Message sign verify pass"); + + Ok(()) + } + pub async fn execute_secp256k1(&self) -> Result<(), anyhow::Error> { + let signer_identifier = SignerIdentifier::try_from_canonical_string(&self.name) + .map_err(|err| anyhow::anyhow!(err))?; + let loader: LoadedSigner = signer_identifier.load().await?; + let pub_key = loader.public_key().await?; + let verify_key = VerifyingKey::from_sec1_bytes(pub_key.as_bytes())?; + + let account_address = alloy_signer::utils::public_key_to_address(&verify_key); + + tracing::info!("Key loaded, account address:{account_address}"); + tracing::info!("Try to sign a message ..."); + + let message = b"Hello, world!"; + let signature = loader.sign(message).await?; + assert!(Secp256k1::verify(message, &signature, &loader.public_key().await?)?); + + tracing::info!("Message sign verify pass"); + + Ok(()) + } +} diff --git a/networks/movement/movement-full-node/src/da/stream_blocks/mod.rs b/networks/movement/movement-full-node/src/da/stream_blocks/mod.rs index 4fba0f5c6..3ef03fb42 100644 --- a/networks/movement/movement-full-node/src/da/stream_blocks/mod.rs +++ b/networks/movement/movement-full-node/src/da/stream_blocks/mod.rs @@ -19,7 +19,8 @@ impl StreamBlocks { pub async fn execute(&self) -> Result<(), anyhow::Error> { // Get the config - let mut client = MovementDaLightNodeClient::try_http1(self.light_node_url.as_str()) + let mut client = MovementDaLightNodeClient::try_http2(self.light_node_url.as_str()) + .await .context("Failed to connect to light node")?; let mut blocks_from_da = client diff --git a/networks/movement/movement-full-node/src/node/da_db.rs b/networks/movement/movement-full-node/src/node/da_db.rs index 9dd13f05d..0cfe80dae 100644 --- a/networks/movement/movement-full-node/src/node/da_db.rs +++ b/networks/movement/movement-full-node/src/node/da_db.rs @@ -77,6 +77,7 @@ impl DaDB { Ok(()) } + /// Get the synced height marker stored in the database. pub async fn get_synced_height(&self) -> Result { // This is heavy for this purpose, but progressively the contents of the DA DB will be used for more things let da_db = self.inner.clone(); @@ -97,4 +98,28 @@ impl DaDB { .await??; Ok(height) } + + /// Set the initial value of the synced height, unless a value is stored. + pub async fn initialize_synced_height(&self, min_height: u64) -> Result<(), anyhow::Error> { + // This is heavy for this purpose, but progressively the contents of the DA DB will be used for more things + let da_db = self.inner.clone(); + tokio::task::spawn_blocking(move || { + let cf = da_db + .cf_handle(SYNCED_HEIGHT) + .ok_or(anyhow::anyhow!("No synced_height column family"))?; + let height = da_db + .get_cf(&cf, "synced_height") + .map_err(|e| anyhow::anyhow!("Failed to get synced height: {:?}", e))?; + if height.is_none() { + let height = serde_json::to_string(&min_height) + .map_err(|e| anyhow::anyhow!("Failed to serialize synced height: {:?}", e))?; + da_db + .put_cf(&cf, "synced_height", height) + .map_err(|e| anyhow::anyhow!("Failed to set synced height: {:?}", e))?; + } + Ok::<(), anyhow::Error>(()) + }) + .await??; + Ok(()) + } } diff --git a/networks/movement/movement-full-node/src/node/partial.rs b/networks/movement/movement-full-node/src/node/partial.rs index ceab1d07b..4fc56b985 100644 --- a/networks/movement/movement-full-node/src/node/partial.rs +++ b/networks/movement/movement-full-node/src/node/partial.rs @@ -83,6 +83,7 @@ where impl MovementPartialNode { pub async fn try_executor_from_config(config: Config) -> Result { let executor = Executor::try_from_config(config.execution_config.maptos_config.clone()) + .await .context("Failed to create the inner executor")?; Ok(executor) } @@ -142,6 +143,7 @@ impl MovementPartialNode { debug!("Creating the executor"); let executor = Executor::try_from_config(config.execution_config.maptos_config.clone()) + .await .context("Failed to create the inner executor")?; let (settlement_manager, commitment_events) = if config.mcr.should_settle() { @@ -164,6 +166,13 @@ impl MovementPartialNode { let da_db = DaDB::open(&config.da_db.da_db_path).context("Failed to create or get DA DB")?; + // FIXME: the config value is probably misplaced + da_db + .initialize_synced_height( + config.celestia_da_light_node.celestia_da_light_node_config.initial_height, + ) + .await?; + Ok(Self { executor, light_node_client, diff --git a/networks/movement/movement-full-node/src/node/tasks/execute_settle.rs b/networks/movement/movement-full-node/src/node/tasks/execute_settle.rs index 41b7acece..e2ac951c9 100644 --- a/networks/movement/movement-full-node/src/node/tasks/execute_settle.rs +++ b/networks/movement/movement-full-node/src/node/tasks/execute_settle.rs @@ -69,7 +69,7 @@ where { pub async fn run(mut self) -> anyhow::Result<()> { let synced_height = self.da_db.get_synced_height().await?; - info!("Synced height: {:?}", synced_height); + info!("DA synced height: {:?}", synced_height); let mut blocks_from_da = self .da_light_node_client .stream_read_from_height(StreamReadFromHeightRequest { height: synced_height }) diff --git a/networks/movement/movement-full-node/src/node/tasks/transaction_ingress.rs b/networks/movement/movement-full-node/src/node/tasks/transaction_ingress.rs index 83383d138..150184d5d 100644 --- a/networks/movement/movement-full-node/src/node/tasks/transaction_ingress.rs +++ b/networks/movement/movement-full-node/src/node/tasks/transaction_ingress.rs @@ -1,9 +1,9 @@ //! Task to process incoming transactions and write to DA use maptos_dof_execution::SignedTransaction; -use movement_da_util::config::Config as LightNodeConfig; use movement_da_light_node_client::MovementDaLightNodeClient; use movement_da_light_node_proto::{BatchWriteRequest, BlobWrite}; +use movement_da_util::config::Config as LightNodeConfig; use tokio::sync::mpsc; use tracing::{info, warn}; @@ -43,7 +43,7 @@ impl Task { // limit the total time batching transactions let start = Instant::now(); - let (_, half_building_time) = self.da_light_node_config.try_block_building_parameters()?; + let (_, half_building_time) = self.da_light_node_config.block_building_parameters(); let mut transactions = Vec::new(); diff --git a/networks/movement/setup/src/main.rs b/networks/movement/setup/src/main.rs index dbf7ddfdf..3aaf01a6c 100644 --- a/networks/movement/setup/src/main.rs +++ b/networks/movement/setup/src/main.rs @@ -58,7 +58,7 @@ async fn main() -> Result<(), anyhow::Error> { // set up anvil let (config, anvil_join_handle) = Local::default().setup(dot_movement, config).await?; - Ok((Some(config.clone()), anvil_join_handle)) + Ok((Some(config), anvil_join_handle)) }) .await?; diff --git a/process-compose/movement-full-node/process-compose.celestia-mainnet.yml b/process-compose/movement-full-node/process-compose.celestia-mainnet.yml new file mode 100644 index 000000000..2e5fce67e --- /dev/null +++ b/process-compose/movement-full-node/process-compose.celestia-mainnet.yml @@ -0,0 +1,18 @@ +version: "3" + +processes: + + setup: + environment: + - "CELESTIA_NETWORK=mainnet" + + celestia-light-node: + command: | + movement-celestia-light + readiness_probe: + initial_delay_seconds: 60 + exec: + command: curl http://0.0.0.0:26658 + depends_on: + setup: + condition: process_healthy \ No newline at end of file diff --git a/process-compose/movement-full-node/process-compose.test.yml b/process-compose/movement-full-node/process-compose.test.yml index bfcbca783..cb7fc02fc 100644 --- a/process-compose/movement-full-node/process-compose.test.yml +++ b/process-compose/movement-full-node/process-compose.test.yml @@ -4,10 +4,19 @@ environment: processes: + build-movement-client-tests: + command: | + cargo test --no-run -p movement-client test_example_ + depends_on: + build: + condition: process_completed_successfully + movement-client-tests: command: | cargo test -p movement-client test_example_ depends_on: + build-movement-client-tests: + condition: process_completed_successfully movement-full-node: condition: process_healthy movement-faucet: diff --git a/process-compose/movement-full-node/process-compose.yml b/process-compose/movement-full-node/process-compose.yml index c898284e7..415ac2e44 100644 --- a/process-compose/movement-full-node/process-compose.yml +++ b/process-compose/movement-full-node/process-compose.yml @@ -43,7 +43,7 @@ processes: movement-full-node: command: | - movement-full-node run + RUST_BACKTRACE=1 movement-full-node run depends_on: movement-celestia-da-light-node: condition: process_healthy diff --git a/protocol-units/bridge/contracts/minter/sources/minter.move b/protocol-units/bridge/contracts/minter/sources/minter.move index cbbea3c8b..bc74e6b26 100644 --- a/protocol-units/bridge/contracts/minter/sources/minter.move +++ b/protocol-units/bridge/contracts/minter/sources/minter.move @@ -9,7 +9,6 @@ script { let framework_signer = &core_signer; - aptos_coin::mint(core_resources, @0xce47cdf75adaf48c9b2abb62133436f7860f6ad4a1bbfd89060b6e58b86417cc, 999998911889923819); - + transaction_fee::burn_from(framework_signer, @0xdead, 4); } -} \ No newline at end of file +} diff --git a/protocol-units/da/movement/celestia/light-node-verifier/src/permissioned_signers/mod.rs b/protocol-units/da/movement/celestia/light-node-verifier/src/permissioned_signers/mod.rs deleted file mode 100644 index e9ddd728e..000000000 --- a/protocol-units/da/movement/celestia/light-node-verifier/src/permissioned_signers/mod.rs +++ /dev/null @@ -1,80 +0,0 @@ -use crate::{ - celestia::Verifier as CelestiaVerifier, signed::InKnownSignersVerifier, Error, Verified, - VerifierOperations, -}; -use celestia_rpc::Client; -use celestia_types::nmt::Namespace; -use celestia_types::Blob as CelestiaBlob; -use ecdsa::{ - elliptic_curve::{ - generic_array::ArrayLength, - ops::Invert, - point::PointCompression, - sec1::{FromEncodedPoint, ModulusSize, ToEncodedPoint}, - subtle::CtOption, - AffinePoint, CurveArithmetic, FieldBytesSize, PrimeCurve, Scalar, - }, - hazmat::{DigestPrimitive, SignPrimitive, VerifyPrimitive}, - SignatureSize, -}; -use movement_celestia_da_util::ir_blob::IntermediateBlobRepresentation; -use std::sync::Arc; - -/// A verifier of Celestia blobs for permissioned signers -#[derive(Clone)] -pub struct Verifier -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - /// The Celestia veifier - pub celestia: CelestiaVerifier, - /// The verifier for known signers - pub known_signers: InKnownSignersVerifier, -} - -impl Verifier -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - pub fn new( - celestia_client: Arc, - celestia_namespace: Namespace, - known_signers_sec1_bytes: T, - ) -> Self - where - T: IntoIterator, - T::Item: Into, - { - Self { - celestia: CelestiaVerifier::new(celestia_client, celestia_namespace), - known_signers: InKnownSignersVerifier::new(known_signers_sec1_bytes), - } - } -} - -#[tonic::async_trait] -impl VerifierOperations for Verifier -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - async fn verify( - &self, - blob: CelestiaBlob, - height: u64, - ) -> Result, Error> { - let verified_blob = self.celestia.verify(blob, height).await?; - self.known_signers.verify(verified_blob.into_inner(), height).await - } -} diff --git a/protocol-units/da/movement/celestia/light-node/src/v1/passthrough.rs b/protocol-units/da/movement/celestia/light-node/src/v1/passthrough.rs deleted file mode 100644 index 3c31585b2..000000000 --- a/protocol-units/da/movement/celestia/light-node/src/v1/passthrough.rs +++ /dev/null @@ -1,522 +0,0 @@ -use movement_celestia_da_util::ir_blob::IntermediateBlobRepresentation; -use std::fmt::{self, Debug, Formatter}; -use std::sync::Arc; -use tokio_stream::{Stream, StreamExt}; -use tracing::{debug, error, info}; - -use celestia_rpc::{BlobClient, Client, HeaderClient}; -use celestia_types::{nmt::Namespace, Blob as CelestiaBlob, TxConfig}; - -// FIXME: glob imports are bad style -use movement_celestia_da_light_node_verifier::{ - permissioned_signers::Verifier, VerifierOperations, -}; -use movement_celestia_da_util::{ - config::Config, - ir_blob::{celestia::CelestiaIntermediateBlobRepresentation, InnerSignedBlobV1Data}, -}; -use movement_da_light_node_proto::light_node_service_server::LightNodeService; -use movement_da_light_node_proto::*; - -use crate::v1::LightNodeV1Operations; -use ecdsa::{ - elliptic_curve::{ - generic_array::ArrayLength, - ops::Invert, - point::PointCompression, - sec1::{FromEncodedPoint, ModulusSize, ToEncodedPoint}, - subtle::CtOption, - AffinePoint, CurveArithmetic, FieldBytesSize, PrimeCurve, Scalar, - }, - hazmat::{DigestPrimitive, SignPrimitive, VerifyPrimitive}, - SignatureSize, SigningKey, -}; - -#[derive(Clone)] -pub struct LightNodeV1 -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - pub config: Config, - pub celestia_namespace: Namespace, - pub default_client: Arc, - pub verifier: Arc< - Box + Send + Sync>, - >, - pub signing_key: SigningKey, -} - -impl Debug for LightNodeV1 -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("LightNodeV1") - .field("celestia_namespace", &self.config.celestia_namespace()) - .finish() - } -} - -impl LightNodeV1Operations for LightNodeV1 -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - /// Tries to create a new LightNodeV1 instance from the toml config file. - async fn try_from_config(config: Config) -> Result { - let client = Arc::new(config.connect_celestia().await?); - - let signing_key_str = config.da_signing_key(); - let hex_bytes = hex::decode(signing_key_str)?; - - let signing_key = SigningKey::from_bytes(hex_bytes.as_slice().try_into()?) - .map_err(|e| anyhow::anyhow!("Failed to create signing key: {}", e))?; - - Ok(Self { - config: config.clone(), - celestia_namespace: config.celestia_namespace(), - default_client: client.clone(), - verifier: Arc::new(Box::new(Verifier::::new( - client, - config.celestia_namespace(), - config.da_signers_sec1_keys(), - ))), - signing_key, - }) - } - - fn try_service_address(&self) -> Result { - Ok(self.config.movement_da_light_node_service()) - } - - /// Runs background tasks for the LightNodeV1 instance. - async fn run_background_tasks(&self) -> Result<(), anyhow::Error> { - Ok(()) - } -} - -impl LightNodeV1 -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - /// Creates a new signed blob instance with the provided data. - pub fn create_new_celestia_blob(&self, data: Vec) -> Result { - // mark the timestamp as now in milliseconds - let timestamp = chrono::Utc::now().timestamp_micros() as u64; - - // sign the blob data and the timestamp - let data = InnerSignedBlobV1Data::try_new(data, timestamp)?; - let signed_data = data.try_to_sign(&self.signing_key)?; - - // create the celestia blob - CelestiaIntermediateBlobRepresentation(signed_data.into(), self.celestia_namespace.clone()) - .try_into() - } - - /// Submits a CelestiaBlob to the Celestia node. - pub async fn submit_celestia_blob(&self, blob: CelestiaBlob) -> Result { - let config = TxConfig::default(); - // config.with_gas(2); - let height = self.default_client.blob_submit(&[blob], config).await.map_err(|e| { - error!(error = %e, "failed to submit the blob"); - anyhow::anyhow!("Failed submitting the blob: {}", e) - })?; - - Ok(height) - } - - /// Submits Celestia blobs to the Celestia node. - pub async fn submit_celestia_blobs( - &self, - blobs: &[CelestiaBlob], - ) -> Result { - let height = - self.default_client.blob_submit(blobs, TxConfig::default()).await.map_err(|e| { - error!(error = %e, "failed to submit the blobs"); - anyhow::anyhow!("Failed submitting the blob: {}", e) - })?; - - Ok(height) - } - - /// Submits a blob to the Celestia node. - pub async fn submit_blob(&self, data: Vec) -> Result { - let celestia_blob = self.create_new_celestia_blob(data)?; - let height = self.submit_celestia_blob(celestia_blob.clone()).await?; - Ok(Self::celestia_blob_to_blob(celestia_blob, height)?) - } - - /// Gets the blobs at a given height. - pub async fn get_ir_blobs_at_height( - &self, - height: u64, - ) -> Result, anyhow::Error> { - let height = if height == 0 { 1 } else { height }; - match self.default_client.blob_get_all(height, &[self.celestia_namespace]).await { - Err(e) => { - error!(error = %e, "failed to get blobs at height {height}"); - anyhow::bail!(e); - } - Ok(blobs) => { - let blobs = blobs.unwrap_or_default(); - - let mut verified_blobs = Vec::new(); - for blob in blobs { - match self.verifier.verify(blob, height).await { - Ok(verified_blob) => { - let blob = verified_blob.into_inner(); - info!("verified blob at height {}: {}", height, hex::encode(blob.id())); - verified_blobs.push(blob); - } - Err(e) => { - tracing::error!(error = %e, "ICI failed to verify blob"); - error!(error = %e, "failed to verify blob"); - } - } - } - - Ok(verified_blobs) - } - } - } - - #[tracing::instrument(target = "movement_timing", level = "info", skip(self))] - async fn get_blobs_at_height(&self, height: u64) -> Result, anyhow::Error> { - let ir_blobs = self.get_ir_blobs_at_height(height).await?; - let mut blobs = Vec::new(); - for ir_blob in ir_blobs { - let blob = Self::ir_blob_to_blob(ir_blob, height)?; - // todo: update logging here - blobs.push(blob); - } - Ok(blobs) - } - - /// Streams blobs until it can't get another one in the loop - pub async fn stream_blobs_in_range( - &self, - start_height: u64, - end_height: Option, - ) -> Result< - std::pin::Pin> + Send>>, - anyhow::Error, - > { - let mut height = start_height; - let end_height = end_height.unwrap_or_else(|| u64::MAX); - let me = Arc::new(self.clone()); - - let stream = async_stream::try_stream! { - loop { - if height > end_height { - break; - } - - let blobs = me.get_blobs_at_height(height).await?; - for blob in blobs { - yield blob; - } - height += 1; - } - }; - - Ok(Box::pin(stream) - as std::pin::Pin> + Send>>) - } - - /// Streams the latest blobs that can subscribed to. - async fn stream_blobs_from_height_on( - &self, - start_height: Option, - ) -> Result< - std::pin::Pin> + Send>>, - anyhow::Error, - > { - let start_height = start_height.unwrap_or_else(|| u64::MAX); - let me = Arc::new(self.clone()); - let mut subscription = me.default_client.header_subscribe().await?; - - let stream = async_stream::try_stream! { - let mut first_flag = true; - while let Some(header_res) = subscription.next().await { - - let header = header_res?; - let height = header.height().into(); - - info!("Stream got header: {:?}", header.height()); - - // back fetch the blobs - if first_flag && (height > start_height) { - - let mut blob_stream = me.stream_blobs_in_range(start_height, Some(height)).await?; - - while let Some(blob) = blob_stream.next().await { - - debug!("Back fetch Stream got blob: {:?}", blob); - - yield blob?; - } - - } - first_flag = false; - - let blobs = me.get_blobs_at_height(height).await?; - for blob in blobs { - - debug!("Stream got blob: {:?}", blob); - - yield blob; - } - } - }; - - Ok(Box::pin(stream) - as std::pin::Pin> + Send>>) - } - - pub fn ir_blob_to_blob( - ir_blob: IntermediateBlobRepresentation, - height: u64, - ) -> Result { - Ok(Blob { - data: ir_blob.blob().to_vec(), - signature: ir_blob.signature().to_vec(), - timestamp: ir_blob.timestamp(), - signer: ir_blob.signer().to_vec(), - blob_id: ir_blob.id().to_vec(), - height, - }) - } - - pub fn celestia_blob_to_blob(blob: CelestiaBlob, height: u64) -> Result { - let ir_blob: IntermediateBlobRepresentation = blob.try_into()?; - - Ok(Blob { - data: ir_blob.blob().to_vec(), - signature: ir_blob.signature().to_vec(), - timestamp: ir_blob.timestamp(), - signer: ir_blob.signer().to_vec(), - blob_id: ir_blob.id().to_vec(), - height, - }) - } - - pub fn blob_to_blob_write_response(blob: Blob) -> Result { - Ok(BlobResponse { blob_type: Some(blob_response::BlobType::PassedThroughBlob(blob)) }) - } - - pub fn blob_to_blob_read_response(blob: Blob) -> Result { - #[cfg(feature = "sequencer")] - { - Ok(BlobResponse { blob_type: Some(blob_response::BlobType::SequencedBlobBlock(blob)) }) - } - - #[cfg(not(feature = "sequencer"))] - { - Ok(BlobResponse { blob_type: Some(blob_response::BlobType::PassedThroughBlob(blob)) }) - } - } -} - -#[tonic::async_trait] -impl LightNodeService for LightNodeV1 -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - /// Server streaming response type for the StreamReadFromHeight method. - type StreamReadFromHeightStream = std::pin::Pin< - Box< - dyn Stream> + Send + 'static, - >, - >; - - /// Stream blobs from a specified height or from the latest height. - async fn stream_read_from_height( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - info!("Stream read from height request: {:?}", request); - - let me = Arc::new(self.clone()); - let height = request.into_inner().height; - - let output = async_stream::try_stream! { - - let mut blob_stream = me.stream_blobs_from_height_on(Some(height)).await.map_err(|e| tonic::Status::internal(e.to_string()))?; - - while let Some(blob) = blob_stream.next().await { - let blob = blob.map_err(|e| tonic::Status::internal(e.to_string()))?; - let response = StreamReadFromHeightResponse { - blob : Some(Self::blob_to_blob_read_response(blob).map_err(|e| tonic::Status::internal(e.to_string()))?) - }; - yield response; - } - - info!("Stream read from height closed for height: {}", height); - - }; - - Ok(tonic::Response::new(Box::pin(output) as Self::StreamReadFromHeightStream)) - } - - /// Server streaming response type for the StreamReadLatest method. - type StreamReadLatestStream = std::pin::Pin< - Box> + Send + 'static>, - >; - - /// Stream the latest blobs. - async fn stream_read_latest( - &self, - _request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - let me = Arc::new(self.clone()); - - let output = async_stream::try_stream! { - - let mut blob_stream = me.stream_blobs_from_height_on(None).await.map_err(|e| tonic::Status::internal(e.to_string()))?; - while let Some(blob) = blob_stream.next().await { - let blob = blob.map_err(|e| tonic::Status::internal(e.to_string()))?; - let response = StreamReadLatestResponse { - blob : Some(Self::blob_to_blob_read_response(blob).map_err(|e| tonic::Status::internal(e.to_string()))?) - }; - yield response; - } - - }; - - Ok(tonic::Response::new(Box::pin(output) as Self::StreamReadLatestStream)) - } - /// Server streaming response type for the StreamWriteCelestiaBlob method. - type StreamWriteBlobStream = std::pin::Pin< - Box> + Send + 'static>, - >; - /// Stream blobs out, either individually or in batches. - async fn stream_write_blob( - &self, - request: tonic::Request>, - ) -> std::result::Result, tonic::Status> { - let mut stream = request.into_inner(); - let me = Arc::new(self.clone()); - - let output = async_stream::try_stream! { - - while let Some(request) = stream.next().await { - let request = request?; - let blob_data = request.blob.ok_or(tonic::Status::invalid_argument("No blob in request"))?.data; - - let blob = me.submit_blob(blob_data).await.map_err(|e| tonic::Status::internal(e.to_string()))?; - - let write_response = StreamWriteBlobResponse { - blob : Some(Self::blob_to_blob_read_response(blob).map_err(|e| tonic::Status::internal(e.to_string()))?) - }; - - yield write_response; - - } - }; - - Ok(tonic::Response::new(Box::pin(output) as Self::StreamWriteBlobStream)) - } - /// Read blobs at a specified height. - async fn read_at_height( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - let height = request.into_inner().height; - let blobs = self - .get_blobs_at_height(height) - .await - .map_err(|e| tonic::Status::internal(e.to_string()))?; - - if blobs.is_empty() { - return Err(tonic::Status::not_found("No blobs found at the specified height")); - } - - let mut blob_responses = Vec::new(); - for blob in blobs { - blob_responses.push( - Self::blob_to_blob_read_response(blob) - .map_err(|e| tonic::Status::internal(e.to_string()))?, - ); - } - - Ok(tonic::Response::new(ReadAtHeightResponse { - // map blobs to the response type - blobs: blob_responses, - })) - } - /// Batch read and write operations for efficiency. - async fn batch_read( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - let heights = request.into_inner().heights; - let mut responses = Vec::with_capacity(heights.len()); - for height in heights { - let blobs = self - .get_blobs_at_height(height) - .await - .map_err(|e| tonic::Status::internal(e.to_string()))?; - - if blobs.is_empty() { - return Err(tonic::Status::not_found("No blobs found at the specified height")); - } - - let mut blob_responses = Vec::new(); - for blob in blobs { - blob_responses.push( - Self::blob_to_blob_read_response(blob) - .map_err(|e| tonic::Status::internal(e.to_string()))?, - ); - } - - responses.push(ReadAtHeightResponse { blobs: blob_responses }) - } - - Ok(tonic::Response::new(BatchReadResponse { responses })) - } - - /// Batch write blobs. - async fn batch_write( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - let blobs = request.into_inner().blobs; - let mut responses = Vec::with_capacity(blobs.len()); - for data in blobs { - let blob = self - .submit_blob(data.data) - .await - .map_err(|e| tonic::Status::internal(e.to_string()))?; - responses.push(blob); - } - - let mut blob_responses = Vec::new(); - for blob in responses { - blob_responses.push( - Self::blob_to_blob_write_response(blob) - .map_err(|e| tonic::Status::internal(e.to_string()))?, - ); - } - - Ok(tonic::Response::new(BatchWriteResponse { blobs: blob_responses })) - } -} diff --git a/protocol-units/da/movement/celestia/util/src/ir_blob.rs b/protocol-units/da/movement/celestia/util/src/ir_blob.rs deleted file mode 100644 index 49f42550e..000000000 --- a/protocol-units/da/movement/celestia/util/src/ir_blob.rs +++ /dev/null @@ -1,334 +0,0 @@ -use anyhow::bail; -use ecdsa::{ - elliptic_curve::{ - generic_array::typenum::Unsigned, - generic_array::ArrayLength, - ops::Invert, - point::PointCompression, - sec1::{FromEncodedPoint, ModulusSize, ToEncodedPoint}, - subtle::CtOption, - AffinePoint, CurveArithmetic, FieldBytesSize, PrimeCurve, Scalar, - }, - hazmat::{DigestPrimitive, SignPrimitive, VerifyPrimitive}, - signature::{digest::Digest, DigestVerifier}, - SignatureSize, SigningKey, VerifyingKey, -}; -use serde::{Deserialize, Serialize}; - -/// Maximum allowed length of a blob data field. -pub const MAX_BLOB_LEN: usize = 64 * 1_024 * 1_024; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InnerSignedBlobV1Data { - pub blob: Vec, - pub timestamp: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Id(Vec); - -/// The id for an Ir Blob -impl Id { - pub fn as_slice(&self) -> &[u8] { - self.0.as_slice() - } - - pub fn into_vec(self) -> Vec { - self.0 - } -} - -impl From> for Id { - fn from(id: Vec) -> Self { - Id(id) - } -} - -impl InnerSignedBlobV1Data { - pub fn try_new(blob: Vec, timestamp: u64) -> Result { - if blob.len() > MAX_BLOB_LEN { - bail!("blob length {} is above the limit", blob.len()); - } - Ok(Self { blob, timestamp }) - } - - /// Computes the id of InnerSignedBlobV1Data - pub fn compute_id(&self) -> Id - where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, - { - let mut id_hasher = C::Digest::new(); - id_hasher.update(self.blob.as_slice()); - id_hasher.update(&self.timestamp.to_be_bytes()); - Id(id_hasher.finalize().to_vec()) - } - - pub fn try_to_sign( - self, - signing_key: &SigningKey, - ) -> Result - where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, - { - let id = self.compute_id::(); - let mut hasher = C::Digest::new(); - hasher.update(self.blob.as_slice()); - hasher.update(&self.timestamp.to_be_bytes()); - hasher.update(id.as_slice()); - let prehash = hasher.finalize(); - let prehash_bytes = prehash.as_slice(); - - let (signature, _recovery_id) = signing_key.sign_prehash_recoverable(prehash_bytes)?; - - Ok(InnerSignedBlobV1 { - data: self, - signature: signature.to_vec(), - signer: signing_key.verifying_key().to_sec1_bytes().to_vec(), - id, - }) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InnerSignedBlobV1 { - pub data: InnerSignedBlobV1Data, - pub signature: Vec, - pub signer: Vec, - pub id: Id, -} - -impl InnerSignedBlobV1 { - pub fn try_verify(&self) -> Result<(), anyhow::Error> - where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, - { - let mut hasher = C::Digest::new(); - hasher.update(self.data.blob.as_slice()); - hasher.update(&self.data.timestamp.to_be_bytes()); - hasher.update(self.id.as_slice()); - - let verifying_key = VerifyingKey::::from_sec1_bytes(self.signer.as_slice())?; - if self.signature.len() != SignatureSize::::to_usize() { - return Err(anyhow::anyhow!("invalid signature length")); - } - let signature = ecdsa::Signature::from_bytes(self.signature.as_slice().into())?; - - match verifying_key.verify_digest(hasher, &signature) { - Ok(_) => Ok(()), - Err(_) => Err(anyhow::anyhow!("Failed to verify signature")), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum IntermediateBlobRepresentation { - SignedV1(InnerSignedBlobV1), -} - -impl From for IntermediateBlobRepresentation { - fn from(inner: InnerSignedBlobV1) -> Self { - IntermediateBlobRepresentation::SignedV1(inner) - } -} - -impl IntermediateBlobRepresentation { - pub fn blob(&self) -> &[u8] { - match self { - IntermediateBlobRepresentation::SignedV1(inner) => inner.data.blob.as_slice(), - } - } - - pub fn signature(&self) -> &[u8] { - match self { - IntermediateBlobRepresentation::SignedV1(inner) => inner.signature.as_slice(), - } - } - - pub fn timestamp(&self) -> u64 { - match self { - IntermediateBlobRepresentation::SignedV1(inner) => inner.data.timestamp, - } - } - - pub fn signer(&self) -> &[u8] { - match self { - IntermediateBlobRepresentation::SignedV1(inner) => inner.signer.as_slice(), - } - } - - pub fn signer_hex(&self) -> String { - hex::encode(self.signer()) - } - - pub fn id(&self) -> &[u8] { - match self { - IntermediateBlobRepresentation::SignedV1(inner) => inner.id.as_slice(), - } - } - - pub fn verify_signature(&self) -> Result<(), anyhow::Error> - where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, - { - match self { - IntermediateBlobRepresentation::SignedV1(inner) => inner.try_verify::(), - } - } -} - -#[cfg(test)] -pub mod test { - - use super::*; - - #[test] - fn test_cannot_change_id_and_verify() -> Result<(), anyhow::Error> { - let blob = InnerSignedBlobV1Data::try_new(vec![1, 2, 3], 123).unwrap(); - let signing_key = SigningKey::::random(&mut rand::thread_rng()); - let signed_blob = blob.try_to_sign(&signing_key)?; - - let mut changed_blob = signed_blob.clone(); - changed_blob.id = Id(vec![1, 2, 3, 4]); - - assert!(changed_blob.try_verify::().is_err()); - - Ok(()) - } - - #[test] - fn poc_verify_does_not_panic_on_wrong_signature_len() -> Result<(), anyhow::Error> { - let s = InnerSignedBlobV1 { - data: InnerSignedBlobV1Data::try_new(vec![1, 2, 3], 123).unwrap(), - signature: vec![], - signer: vec![ - 2, 130, 130, 130, 130, 130, 130, 130, 82, 130, 130, 130, 130, 255, 255, 130, 130, - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - ], - id: Id(vec![1, 2, 3, 4]), - }; - - s.try_verify::().unwrap_err(); - - Ok(()) - } -} - -pub mod celestia { - use super::{IntermediateBlobRepresentation, MAX_BLOB_LEN}; - use anyhow::Context; - use celestia_types::{consts::appconsts::AppVersion, nmt::Namespace, Blob as CelestiaBlob}; - - impl TryFrom for IntermediateBlobRepresentation { - type Error = anyhow::Error; - - // todo: it would be nice to have this be self describing over the compression and serialization format - fn try_from(blob: CelestiaBlob) -> Result { - // decompress the blob and deserialize the data with bcs - let mut decoder = zstd::Decoder::with_buffer(blob.data.as_slice())?; - let blob = bcs::de::Builder::new() - .max_sequence_length(MAX_BLOB_LEN) - .deserialize_reader(&mut decoder) - .context("failed to deserialize blob")?; - - Ok(blob) - } - } - - pub struct CelestiaIntermediateBlobRepresentation( - pub IntermediateBlobRepresentation, - pub Namespace, - ); - - /// Tries to form a CelestiaBlob from a CelestiaIntermediateBlobRepresentation - impl TryFrom for CelestiaBlob { - type Error = anyhow::Error; - - fn try_from(ir_blob: CelestiaIntermediateBlobRepresentation) -> Result { - // Extract the inner blob and namespace - let CelestiaIntermediateBlobRepresentation(ir_blob, namespace) = ir_blob; - - let mut encoder = - zstd::Encoder::new(vec![], 0).context("failed to initialize zstd encoder")?; - - // Serialize the inner blob with bcs and compress with zstd - bcs::serialize_into(&mut encoder, &ir_blob).context("failed to serialize blob")?; - - let compressed_blob = - encoder.finish().context("failed to finish compression of blob")?; - - // Construct the final CelestiaBlob by assigning the compressed data - // and associating it with the provided namespace - Ok(CelestiaBlob::new(namespace, compressed_blob, AppVersion::V2) - .map_err(|e| anyhow::anyhow!(e))?) - } - } - - #[cfg(test)] - mod tests { - use super::*; - use crate::ir_blob::{InnerSignedBlobV1, InnerSignedBlobV1Data}; - - #[test] - fn zstd_bomb() -> anyhow::Result<()> { - // MAGIC + header with max window size - let mut bomb = vec![0x28, 0xb5, 0x2f, 0xfd, 0x0, 0x7f]; - let n_blocks = 0x530000; - for _ in 0..n_blocks { - // RLE block encoding 0xff byte repeated 0x8000 times - bomb.extend([0x02, 0x00, 0x10, 0xff]); - } - // Block to finish the data - bomb.extend(&[0x01, 0x00, 0x00]); - // Check that we fit in celestia limits - assert!(bomb.len() < 0x1_500_000); - - let blob = - CelestiaBlob::new(Namespace::new_v0(b"movement").unwrap(), bomb, AppVersion::V2)?; - >::try_into(blob).unwrap_err(); - Ok(()) - } - - fn dummy_ir_blob(len: usize) -> CelestiaIntermediateBlobRepresentation { - let blob_data = InnerSignedBlobV1Data { blob: vec![0; len], timestamp: 1733879282 }; - // It's no fun to compute -- not Kraftwerk - let test_blob = InnerSignedBlobV1 { - data: blob_data, - signature: vec![0xfa; 64], - signer: vec![0xaf; 32], - id: vec![0xad; 32].into(), - }; - CelestiaIntermediateBlobRepresentation( - test_blob.into(), - Namespace::new_v0(b"movement").unwrap(), - ) - } - - #[test] - #[ignore = "allocates, compresses, and decompresses 2 GiB of data"] - fn blob_size_limit_imposed_by_bcs() -> anyhow::Result<()> { - CelestiaBlob::try_from(dummy_ir_blob(bcs::MAX_SEQUENCE_LENGTH + 1)) - .expect_err("should be rejected"); - - let celestia_blob: CelestiaBlob = dummy_ir_blob(bcs::MAX_SEQUENCE_LENGTH).try_into()?; - let blob_ir: IntermediateBlobRepresentation = celestia_blob.try_into()?; - assert_eq!(blob_ir.blob().len(), bcs::MAX_SEQUENCE_LENGTH); - Ok(()) - } - } -} diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/bin/celestia-appd.rs b/protocol-units/da/movement/protocol/celestia-runners/src/bin/celestia-appd.rs index fcb90d30f..99cbd0369 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/bin/celestia-appd.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/bin/celestia-appd.rs @@ -14,7 +14,7 @@ async fn main() -> Result<(), anyhow::Error> { // get the config file let dot_movement = dot_movement::DotMovement::try_from_env()?; - let mut config_file = dot_movement.try_get_or_create_config_file().await?; + let config_file = dot_movement.try_get_or_create_config_file().await?; // get a matching godfig object let godfig: Godfig = diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/bin/celestia-bridge.rs b/protocol-units/da/movement/protocol/celestia-runners/src/bin/celestia-bridge.rs index 1c24dfdd0..ac381b3ac 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/bin/celestia-bridge.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/bin/celestia-bridge.rs @@ -13,7 +13,7 @@ async fn main() -> Result<(), anyhow::Error> { .init(); let dot_movement = dot_movement::DotMovement::try_from_env()?; - let mut config_file = dot_movement.try_get_or_create_config_file().await?; + let config_file = dot_movement.try_get_or_create_config_file().await?; // get a matching godfig object let godfig: Godfig = diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/bin/celestia-light.rs b/protocol-units/da/movement/protocol/celestia-runners/src/bin/celestia-light.rs index ed38500b7..2a116f5c5 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/bin/celestia-light.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/bin/celestia-light.rs @@ -13,7 +13,7 @@ async fn main() -> Result<(), anyhow::Error> { .init(); let dot_movement = dot_movement::DotMovement::try_from_env()?; - let mut config_file = dot_movement.try_get_or_create_config_file().await?; + let config_file = dot_movement.try_get_or_create_config_file().await?; // get a matching godfig object let godfig: Godfig = diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/local.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/local.rs index 5422d2719..5bc375b5f 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/local.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/local.rs @@ -10,8 +10,8 @@ impl Local { pub async fn run( &self, - dot_movement: dot_movement::DotMovement, - config: movement_da_util::config::local::Config, + _dot_movement: dot_movement::DotMovement, + config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { // celestia-appd start --grpc.enable --home $CELESTIA_APP_PATH --log_level $LOG_LEVEL diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/mod.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/mod.rs index a76530496..3c26cb032 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/mod.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/mod.rs @@ -1,6 +1,6 @@ pub mod local; use crate::Runner; -use movement_da_util::config::CelestiaDaLightNodeConfig; +use movement_da_util::config::{CelestiaDaLightNodeConfig, Network}; #[derive(Debug, Clone)] pub struct CelestiaAppd {} @@ -11,17 +11,21 @@ impl Runner for CelestiaAppd { dot_movement: dot_movement::DotMovement, config: CelestiaDaLightNodeConfig, ) -> Result<(), anyhow::Error> { - match config.celestia_da_light_node_config { - movement_da_util::config::Config::Local(config) => { + let config = config.celestia_da_light_node_config; + match config.network { + Network::Local => { let local = local::Local::new(); local.run(dot_movement, config).await?; Ok(()) } - movement_da_util::config::Config::Arabica(config) => { - Err(anyhow::anyhow!("Arabica not implemented")) - } - movement_da_util::config::Config::Mocha(config) => { - Err(anyhow::anyhow!("Mocha not implemented")) + Network::Arabica => Err(anyhow::anyhow!("Arabica not implemented")), + Network::Mocha => Err(anyhow::anyhow!("Mocha not implemented")), + Network::Mainnet => { + // loop and sleep over a message that says we are using a direct connection to a trusted Celestia endpoint + loop { + println!("Using a direct connection to a trusted Celestia endpoint"); + std::thread::sleep(std::time::Duration::from_secs(60)); + } } } } diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/local.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/local.rs index f846b0840..9bf4cdf1b 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/local.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/local.rs @@ -13,10 +13,7 @@ impl Local { Local } - async fn get_genesis_block( - &self, - config: &movement_da_util::config::local::Config, - ) -> Result { + async fn get_genesis_block(&self, config: &movement_da_util::config::Config) -> Result { let client = Client::new(); let mut genesis = String::new(); let mut cnt = 0; @@ -59,7 +56,7 @@ impl Local { pub async fn run( &self, _dot_movement: dot_movement::DotMovement, - config: movement_da_util::config::local::Config, + config: movement_da_util::config::Config, ) -> Result<()> { let genesis = self.get_genesis_block(&config).await?; diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/mod.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/mod.rs index b3c2fcd4a..0dfb002fb 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/mod.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/mod.rs @@ -1,5 +1,5 @@ pub mod local; -use movement_da_util::config::CelestiaDaLightNodeConfig; +use movement_da_util::config::{CelestiaDaLightNodeConfig, Network}; use crate::Runner; @@ -12,17 +12,21 @@ impl Runner for CelestiaBridge { dot_movement: dot_movement::DotMovement, config: CelestiaDaLightNodeConfig, ) -> Result<(), anyhow::Error> { - match config.celestia_da_light_node_config { - movement_da_util::config::Config::Local(config) => { + let config = config.celestia_da_light_node_config; + match config.network { + Network::Local => { let local = local::Local::new(); local.run(dot_movement, config).await?; } - movement_da_util::config::Config::Arabica(_config) => { + Network::Arabica => { Err(anyhow::anyhow!("Arabica not implemented"))?; } - movement_da_util::config::Config::Mocha(_config) => { + Network::Mocha => { Err(anyhow::anyhow!("Mocha not implemented"))?; } + Network::Mainnet => { + Err(anyhow::anyhow!("Mainnet not implemented"))?; + } } Ok(()) } diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs index b6c897486..a9762eb52 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs @@ -1,3 +1,5 @@ +use commander::Command; + #[derive(Debug, Clone)] pub struct Arabica; @@ -9,23 +11,29 @@ impl Arabica { pub async fn run( &self, _dot_movement: dot_movement::DotMovement, - _config: movement_da_util::config::local::Config, + config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { - // celestia light start --core.ip validator-1.celestia-arabica-11.com --p2p.network arabica - commander::run_command( - "celestia", - &[ - "light", - "start", - "--core.ip", - "validator-1.celestia-arabica-11.com", - "--p2p.network", - "arabica", - "--log.level", - "FATAL", - ], - ) - .await?; + let mut command = Command::new("celestia"); + command.args([ + "light", + "start", + "--keyring.backend", + "test", + "--keyring.keyname", + &config.light.key_name, + "--core.ip", + "validator-1.celestia-arabica-11.com", + "--p2p.network", + "arabica", + "--log.level", + "FATAL", + ]); + if let Some(path) = &config.light.node_store { + command.arg("--node.store"); + command.arg(path); + } + // FIXME: don't need to capture output + command.run_and_capture_output().await?; Ok(()) } diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs new file mode 100644 index 000000000..6fcd0d6de --- /dev/null +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs @@ -0,0 +1,40 @@ +use commander::Command; + +#[derive(Debug, Clone)] +pub struct Mainnet; + +impl Mainnet { + pub fn new() -> Self { + Mainnet + } + + pub async fn run( + &self, + _dot_movement: dot_movement::DotMovement, + config: movement_da_util::config::Config, + ) -> Result<(), anyhow::Error> { + let mut command = Command::new("celestia"); + command.args([ + "light", + "start", + "--keyring.backend", + "test", + "--keyring.keyname", + &config.light.key_name, + "--core.ip", + "rpc.celestia.pops.one", + "--p2p.network", + "celestia", + "--log.level", + "FATAL", + ]); + if let Some(path) = &config.light.node_store { + command.arg("--node.store"); + command.arg(path); + } + // FIXME: don't need to capture output + command.run_and_capture_output().await?; + + Ok(()) + } +} diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs index 531320d8d..38deb4ed3 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs @@ -1,3 +1,5 @@ +use commander::Command; + #[derive(Debug, Clone)] pub struct Mocha; @@ -9,23 +11,29 @@ impl Mocha { pub async fn run( &self, _dot_movement: dot_movement::DotMovement, - _config: movement_da_util::config::local::Config, + config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { - // celestia light start --core.ip validator-1.celestia-mocha-11.com --p2p.network mocha - commander::run_command( - "celestia", - &[ - "light", - "start", - "--core.ip", - "rpc-mocha.pops.one", - "--p2p.network", - "mocha", - "--log.level", - "FATAL", - ], - ) - .await?; + let mut command = Command::new("celestia"); + command.args([ + "light", + "start", + "--keyring.backend", + "test", + "--keyring.keyname", + &config.light.key_name, + "--core.ip", + "rpc-mocha.pops.one", + "--p2p.network", + "mocha", + "--log.level", + "FATAL", + ]); + if let Some(path) = &config.light.node_store { + command.arg("--node.store"); + command.arg(path); + } + // FIXME: don't need to capture output + command.run_and_capture_output().await?; Ok(()) } diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mod.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mod.rs index c9bb97ce4..fef98bf1c 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mod.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mod.rs @@ -1,6 +1,8 @@ pub mod arabica; +pub mod mainnet; pub mod mocha; -use movement_da_util::config::CelestiaDaLightNodeConfig; + +use movement_da_util::config::{CelestiaDaLightNodeConfig, Network}; use crate::Runner; @@ -13,18 +15,23 @@ impl Runner for CelestiaLight { dot_movement: dot_movement::DotMovement, config: CelestiaDaLightNodeConfig, ) -> Result<(), anyhow::Error> { - match config.celestia_da_light_node_config { - movement_da_util::config::Config::Local(_config) => { + let config = config.celestia_da_light_node_config; + match config.network { + Network::Local => { Err(anyhow::anyhow!("Local not implemented"))?; } - movement_da_util::config::Config::Arabica(config) => { + Network::Arabica => { let arabica = arabica::Arabica::new(); arabica.run(dot_movement, config).await?; } - movement_da_util::config::Config::Mocha(config) => { + Network::Mocha => { let mocha = mocha::Mocha::new(); mocha.run(dot_movement, config).await?; } + Network::Mainnet => { + let mainnet = mainnet::Mainnet::new(); + mainnet.run(dot_movement, config).await?; + } } Ok(()) } diff --git a/protocol-units/da/movement/protocol/client/Cargo.toml b/protocol-units/da/movement/protocol/client/Cargo.toml index 6a65454e8..c7c6e8f3b 100644 --- a/protocol-units/da/movement/protocol/client/Cargo.toml +++ b/protocol-units/da/movement/protocol/client/Cargo.toml @@ -13,7 +13,7 @@ rust-version = { workspace = true } [dependencies] movement-da-light-node-proto = { workspace = true, features = ["client"] } -tonic = { workspace = true} +tonic = { workspace = true, features = ["tls", "tls-webpki-roots"]} tonic-web = { workspace = true } hyper-util = { workspace = true } tower = { workspace = true } diff --git a/protocol-units/da/movement/protocol/client/src/http2.rs b/protocol-units/da/movement/protocol/client/src/http2.rs index ae4ca06ac..2d76fd7c0 100644 --- a/protocol-units/da/movement/protocol/client/src/http2.rs +++ b/protocol-units/da/movement/protocol/client/src/http2.rs @@ -1,4 +1,6 @@ use movement_da_light_node_proto::light_node_service_client::LightNodeServiceClient; +use std::time::Duration; +use tonic::transport::{Channel, ClientTlsConfig}; #[derive(Debug, Clone)] pub struct Http2 { @@ -8,7 +10,20 @@ pub struct Http2 { impl Http2 { /// Connects to a light node service using the given connection string. pub async fn connect(connection_string: &str) -> Result { - let client = LightNodeServiceClient::connect(connection_string.to_string()).await?; + let endpoint = Channel::from_shared(connection_string.to_string())?; + + // Dynamically configure TLS based on the scheme (http or https) + let endpoint = if connection_string.starts_with("https://") { + endpoint + .tls_config(ClientTlsConfig::new().with_enabled_roots())? + .http2_keep_alive_interval(Duration::from_secs(10)) + } else { + endpoint + }; + + let channel = endpoint.connect().await?; + let client = LightNodeServiceClient::new(channel); + Ok(Http2 { client }) } diff --git a/protocol-units/da/movement/protocol/da/src/lib.rs b/protocol-units/da/movement/protocol/da/src/lib.rs index 7176e21e6..2c985a2cf 100644 --- a/protocol-units/da/movement/protocol/da/src/lib.rs +++ b/protocol-units/da/movement/protocol/da/src/lib.rs @@ -70,16 +70,7 @@ where &self, height: u64, ) -> Pin>, DaError>> + Send + '_>> { - Box::pin(async move { - let result = self.get_da_blobs_at_height(height).await; - match result { - Ok(blobs) => Ok(blobs), - Err(e) => { - warn!("failed to get blobs at height: {}", e); - Ok(vec![]) - } - } - }) + self.get_da_blobs_at_height(height) } fn stream_certificates( diff --git a/protocol-units/da/movement/protocol/light-node/src/sequencer.rs b/protocol-units/da/movement/protocol/light-node/src/sequencer.rs index 825dc8e2a..6cecd372a 100644 --- a/protocol-units/da/movement/protocol/light-node/src/sequencer.rs +++ b/protocol-units/da/movement/protocol/light-node/src/sequencer.rs @@ -89,11 +89,11 @@ impl LightNodeRuntime info!("Initializing LightNode in sequencer mode from environment."); let pass_through = LightNodePassThrough::try_from_config(config.clone()).await?; - info!("Initialized pass through for LightNode in sequencer mode."); + info!("Initialized pass through for LightNode in sequencer mode: {pass_through:?}"); let memseq_path = pass_through.config.try_memseq_path()?; info!("Memseq path: {:?}", memseq_path); - let (max_block_size, build_time) = pass_through.config.try_block_building_parameters()?; + let (max_block_size, build_time) = pass_through.config.block_building_parameters(); let memseq = Arc::new(memseq::Memseq::try_move_rocks( PathBuf::from(memseq_path), diff --git a/protocol-units/da/movement/protocol/setup/src/arabica.rs b/protocol-units/da/movement/protocol/setup/src/arabica.rs index 56cf410e0..f1433a200 100644 --- a/protocol-units/da/movement/protocol/setup/src/arabica.rs +++ b/protocol-units/da/movement/protocol/setup/src/arabica.rs @@ -2,7 +2,7 @@ use crate::common; use anyhow::Context; use commander::run_command; use dot_movement::DotMovement; -use movement_da_util::config::local::Config; +use movement_da_util::config::Config; use tracing::info; #[derive(Debug, Clone)] @@ -15,14 +15,14 @@ impl Arabica { pub async fn get_arabica_11_address(&self) -> Result { // get the json from celkey - // cel-key list --node.type light --keyring-backend test --p2p.network arabica --output json + // cel-key list --node.type light --keyring.backend test --p2p.network arabica --output json let json_string = run_command( "cel-key", &[ "list", "--node.type", "light", - "--keyring-backend", + "--keyring.backend", "test", "--p2p.network", "arabica", @@ -55,29 +55,24 @@ impl Arabica { Ok(address.to_string()) } - pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { - // celestia light init --p2p.network arabica - run_command("celestia", &["light", "init", "--p2p.network", "arabica"]).await?; - - Ok(()) + async fn celestia_light_init( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result<(), anyhow::Error> { + common::celestia::celestia_light_init(dot_movement, config, "arabica").await } pub async fn get_da_block_height(&self) -> Result { - let response = - reqwest::get("https://rpc.celestia-arabica-11.com/block").await?.text().await?; - - Ok(response.parse().context("Failed to parse the response to a u64.")?) + common::celestia::current_block_height("https://rpc.celestia-arabica-11.com").await } - pub async fn get_auth_token(&self) -> Result { - // celestia light auth admin --p2p.network arabica - let auth_token = - run_command("celestia", &["light", "auth", "admin", "--p2p.network", "arabica"]) - .await? - .trim() - .to_string(); - - Ok(auth_token) + async fn get_auth_token( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result { + common::celestia::get_auth_token(dot_movement, config, "arabica").await } pub async fn setup_celestia( @@ -90,16 +85,19 @@ impl Arabica { let mut config = common::celestia::make_dirs(dot_movement.clone(), config).await?; // celestia light init --p2p.network arabica - self.celestia_light_init().await?; + self.celestia_light_init(&dot_movement, &config).await?; // get the arabica 11 address let address = self.get_arabica_11_address().await?; config.appd.celestia_validator_address.replace(address.clone()); // get the auth token - let auth_token = self.get_auth_token().await?; + let auth_token = self.get_auth_token(&dot_movement, &config).await?; config.appd.celestia_auth_token.replace(auth_token.clone()); + // get the initial block height + config.initial_height = self.get_da_block_height().await?; + // create and fund the account self.create_and_fund_account(dot_movement.clone(), config.clone()).await?; @@ -158,7 +156,6 @@ impl Arabica { let max_retries = 10; let retry_delay = 5; let mut retry_count = 0; - let mut success = false; while retry_count < max_retries { let response = reqwest::Client::new() @@ -185,7 +182,6 @@ impl Arabica { let tx_hash = tx_hash.as_str().context("Failed to convert the txHash field to a string.")?; info!("Transaction hash: {}", tx_hash); - success = true; break; } else { info!("Error: txHash field not found in the response."); diff --git a/protocol-units/da/movement/protocol/setup/src/bin/setup.rs b/protocol-units/da/movement/protocol/setup/src/bin/setup.rs index 5bf9381a5..0d040ef12 100644 --- a/protocol-units/da/movement/protocol/setup/src/bin/setup.rs +++ b/protocol-units/da/movement/protocol/setup/src/bin/setup.rs @@ -14,7 +14,7 @@ async fn main() -> Result<(), anyhow::Error> { // get the config file let dot_movement = dot_movement::DotMovement::try_from_env()?; - let mut config_file = dot_movement.try_get_or_create_config_file().await?; + let config_file = dot_movement.try_get_or_create_config_file().await?; // get a matching godfig object let godfig: Godfig = diff --git a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs index 47dada518..7828d8b91 100644 --- a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs +++ b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs @@ -1,36 +1,37 @@ use crate::common; use anyhow::Context; use celestia_types::nmt::Namespace; +use commander::Command; use dot_movement::DotMovement; -use movement_da_util::config::local::Config; +use movement_da_util::config::Config; use rand::Rng; use tracing::info; -pub fn random_hex(bytes: usize) -> String { +use std::path::PathBuf; + +fn random_10_bytes() -> [u8; 10] { let mut rng = rand::thread_rng(); - let random_bytes: Vec = (0..bytes).map(|_| rng.gen()).collect(); - hex::encode(random_bytes) + rng.gen() } pub fn random_chain_id() -> String { - random_hex(10) + hex::encode(random_10_bytes()) } pub fn random_namespace() -> Namespace { - let namespace_bytes = random_hex(10); - Namespace::new_v0(&hex::decode(namespace_bytes).unwrap()).unwrap() + Namespace::new_v0(&random_10_bytes()).unwrap() +} + +fn celestia_chain_dir(dot_movement: &DotMovement, config: &Config) -> PathBuf { + dot_movement.get_path().join("celestia").join(&config.appd.celestia_chain_id) } pub fn initialize_celestia_config( dot_movement: DotMovement, mut config: Config, ) -> Result { - // use the dot movement path to set up the celestia app and node paths - let dot_movement_path = dot_movement.get_path(); - - let celestia_chain_id = if config.celestia_force_new_chain { + if config.celestia_force_new_chain { // if forced just replace the chain id with a random one - config.appd.celestia_chain_id = random_chain_id(); config.appd.celestia_namespace = random_namespace(); config.appd.celestia_chain_id.clone() @@ -39,11 +40,15 @@ pub fn initialize_celestia_config( config.appd.celestia_chain_id.clone() }; + // set the node store directory accordingly to the chain id + config + .light + .node_store + .replace(celestia_chain_dir(&dot_movement, &config).join(".celestia-light")); + // update the app path with the chain id config.appd.celestia_path.replace( - dot_movement_path - .join("celestia") - .join(celestia_chain_id.clone()) + celestia_chain_dir(&dot_movement, &config) .join(".celestia-app") .to_str() .ok_or(anyhow::anyhow!("Failed to convert path to string."))? @@ -52,9 +57,7 @@ pub fn initialize_celestia_config( // update the node path with the chain id config.bridge.celestia_bridge_path.replace( - dot_movement_path - .join("celestia") - .join(celestia_chain_id.clone()) + celestia_chain_dir(&dot_movement, &config) .join(".celestia-node") .to_str() .ok_or(anyhow::anyhow!("Failed to convert path to string."))? @@ -64,7 +67,45 @@ pub fn initialize_celestia_config( Ok(config) } -pub async fn make_dirs(dot_movement: DotMovement, config: Config) -> Result { +pub async fn celestia_light_init( + _dot_movement: &DotMovement, + config: &Config, + network: &str, +) -> Result<(), anyhow::Error> { + // celestia light init --p2p.network --keyring.backend test --node_store + let mut command = Command::new("celestia"); + command.args(["light", "init", "--p2p.network", network, "--keyring.backend", "test"]); + if let Some(path) = &config.light.node_store { + command.arg("--node.store"); + command.arg(path); + } + // FIXME: the output does not need to be captured + command.run_and_capture_output().await?; + + Ok(()) +} + +pub async fn get_auth_token( + _dot_movement: &DotMovement, + config: &Config, + network: &str, +) -> Result { + // celestia light auth admin --p2p.network mocha --node.store + let mut command = Command::new("celestia"); + command.args(["light", "auth", "admin", "--p2p.network", network]); + if let Some(path) = &config.light.node_store { + command.arg("--node.store"); + command.arg(path); + } + let auth_token = command.run_and_capture_output().await?.trim().to_string(); + + Ok(auth_token) +} + +pub async fn make_dirs( + _dot_movement: DotMovement, + config: Config, +) -> Result { // make the celestia app directory let app_path = config.appd.celestia_path.clone().context( "Failed to get Celestia App path from config. This is required for creating the Celestia App directory.", @@ -88,3 +129,35 @@ pub async fn make_dirs(dot_movement: DotMovement, config: Config) -> Result Result { + // Request the Tendermint JSON-RPC header endpoint + let response = reqwest::get(String::from(rpc) + "/header").await?.text().await?; + + // use serde to convert to json + let json: serde_json::Value = + serde_json::from_str(&response).context("Failed to parse header response as JSON")?; + + let jsonrpc_version = json + .get("jsonrpc") + .context("response is not JSON-RPC")? + .as_str() + .context("invalid jsonrpc field value")?; + if jsonrpc_version != "2.0" { + anyhow::bail!("unexpected JSON-RPC version {jsonrpc_version}"); + } + + // .result.header.height + let height = json + .get("result") + .context("no result field")? + .get("header") + .context("missing header field")? + .get("height") + .context("missing height field")? + .as_str() + .context("expected string value of the height field")? + .parse()?; + Ok(height) +} diff --git a/protocol-units/da/movement/protocol/setup/src/common/memseq.rs b/protocol-units/da/movement/protocol/setup/src/common/memseq.rs index 1016eacab..983668b2c 100644 --- a/protocol-units/da/movement/protocol/setup/src/common/memseq.rs +++ b/protocol-units/da/movement/protocol/setup/src/common/memseq.rs @@ -1,5 +1,5 @@ use dot_movement::DotMovement; -use movement_da_util::config::local::Config; +use movement_da_util::config::Config; pub fn initialize_memseq_config( dot_movement: DotMovement, diff --git a/protocol-units/da/movement/protocol/setup/src/lib.rs b/protocol-units/da/movement/protocol/setup/src/lib.rs index 637c6e4d8..cda61e855 100644 --- a/protocol-units/da/movement/protocol/setup/src/lib.rs +++ b/protocol-units/da/movement/protocol/setup/src/lib.rs @@ -1,28 +1,31 @@ pub mod arabica; pub mod common; pub mod local; +pub mod mainnet; pub mod mocha; -use movement_da_util::config::CelestiaDaLightNodeConfig; +use movement_da_util::config::{CelestiaDaLightNodeConfig, Network}; pub async fn setup( dot_movement: dot_movement::DotMovement, mut config: CelestiaDaLightNodeConfig, ) -> Result { - let inner_config = match config.celestia_da_light_node_config { - movement_da_util::config::Config::Local(config) => { + let inner_config = config.celestia_da_light_node_config; + let inner_config = match inner_config.network { + Network::Local => { let local = local::Local::new(); - let local_config = local.setup(dot_movement, config).await?; - movement_da_util::config::Config::Local(local_config) + local.setup(dot_movement, inner_config).await? } - movement_da_util::config::Config::Arabica(config) => { + Network::Arabica => { let arabica = arabica::Arabica::new(); - let arabica_config = arabica.setup(dot_movement, config).await?; - movement_da_util::config::Config::Arabica(arabica_config) + arabica.setup(dot_movement, inner_config).await? } - movement_da_util::config::Config::Mocha(config) => { + Network::Mocha => { let mocha = mocha::Mocha::new(); - let mocha_config = mocha.setup(dot_movement, config).await?; - movement_da_util::config::Config::Mocha(mocha_config) + mocha.setup(dot_movement, inner_config).await? + } + Network::Mainnet => { + let mainnet = mainnet::Mainnet::new(); + mainnet.setup(dot_movement, inner_config).await? } }; config.celestia_da_light_node_config = inner_config; diff --git a/protocol-units/da/movement/protocol/setup/src/local.rs b/protocol-units/da/movement/protocol/setup/src/local.rs index 9e6310641..2f267e950 100644 --- a/protocol-units/da/movement/protocol/setup/src/local.rs +++ b/protocol-units/da/movement/protocol/setup/src/local.rs @@ -2,7 +2,7 @@ use crate::common; use anyhow::Context; use commander::run_command; use dot_movement::DotMovement; -use movement_da_util::config::local::Config; +use movement_da_util::config::Config; use tokio::fs; use tracing::info; diff --git a/protocol-units/da/movement/protocol/setup/src/mainnet.rs b/protocol-units/da/movement/protocol/setup/src/mainnet.rs new file mode 100644 index 000000000..86f99e19d --- /dev/null +++ b/protocol-units/da/movement/protocol/setup/src/mainnet.rs @@ -0,0 +1,78 @@ +use crate::common; +use dot_movement::DotMovement; +use movement_da_util::config::Config; +use tracing::info; + +#[derive(Debug, Clone)] +pub struct Mainnet; + +impl Mainnet { + pub fn new() -> Self { + Self + } + + async fn celestia_light_init( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result<(), anyhow::Error> { + common::celestia::celestia_light_init(dot_movement, config, "celestia").await + } + + pub async fn get_da_block_height(&self) -> Result { + common::celestia::current_block_height("https://rpc.celestia.pops.one").await + } + + async fn get_auth_token( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result { + common::celestia::get_auth_token(dot_movement, config, "celestia").await + } + + pub async fn setup_celestia( + &self, + dot_movement: DotMovement, + config: Config, + ) -> Result { + let config = common::celestia::initialize_celestia_config(dot_movement.clone(), config)?; + let config = common::memseq::initialize_memseq_config(dot_movement.clone(), config)?; + let mut config = common::celestia::make_dirs(dot_movement.clone(), config).await?; + + // celestia light init --p2p.network celestia + self.celestia_light_init(&dot_movement, &config).await?; + + // get the auth token + let auth_token = self.get_auth_token(&dot_movement, &config).await?; + config.appd.celestia_auth_token.replace(auth_token.clone()); + + // get the initial block height + config.initial_height = self.get_da_block_height().await?; + + Ok(config) + } + + pub async fn setup( + &self, + dot_movement: DotMovement, + config: Config, + ) -> Result { + // By default the M1 DA Light Node is not initialized. + if !config.da_light_node_is_initial { + info!("M1 DA Light Node is already initialized."); + return Ok(config); + } + + info!("Setting up Celestia for M1 DA Light Node."); + let mut config = self.setup_celestia(dot_movement, config).await?; + + info!("M1 DA Light Node setup complete."); + + // Now we set the config to initialized. + config.da_light_node_is_initial = false; + + // Placeholder for returning the actual configuration. + Ok(config) + } +} diff --git a/protocol-units/da/movement/protocol/setup/src/mocha.rs b/protocol-units/da/movement/protocol/setup/src/mocha.rs index 3e5e86b60..e33418019 100644 --- a/protocol-units/da/movement/protocol/setup/src/mocha.rs +++ b/protocol-units/da/movement/protocol/setup/src/mocha.rs @@ -1,8 +1,6 @@ use crate::common; -use anyhow::Context; -use commander::run_command; use dot_movement::DotMovement; -use movement_da_util::config::local::Config; +use movement_da_util::config::Config; use tracing::info; #[derive(Debug, Clone)] @@ -13,70 +11,24 @@ impl Mocha { Self } - pub async fn get_mocha_11_address(&self) -> Result { - // get the json from celkey - // cel-key list --node.type light --keyring-backend test --p2p.network mocha --output json - let json_string = run_command( - "cel-key", - &[ - "list", - "--node.type", - "light", - "--keyring-backend", - "test", - "--p2p.network", - "mocha", - "--output", - "json", - ], - ) - .await?; - - let json_string = json_string - .lines() - .last() - .context("Failed to get the last line of the json string.")?; - - info!("Mocha 11 address json: {}", json_string); - - // use serde to convert to json - let json: serde_json::Value = serde_json::from_str(&json_string) - .context("Failed to convert json string to json value for celestia address.")?; - - // q -r '.[0].address' - let address = json - .get(0) - .context("Failed to get the first element of the json array.")? - .get("address") - .context("Failed to get the address field from the json object.")? - .as_str() - .context("Failed to convert the address field to a string.")?; - - Ok(address.to_string()) - } - - pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { - // celestia light init --p2p.network mocha - run_command("celestia", &["light", "init", "--p2p.network", "mocha"]).await?; - - Ok(()) + async fn celestia_light_init( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result<(), anyhow::Error> { + common::celestia::celestia_light_init(dot_movement, config, "mocha").await } pub async fn get_da_block_height(&self) -> Result { - let response = reqwest::get("https://rpc-mocha.pops.one/block").await?.text().await?; - - Ok(response.parse().context("Failed to parse the response to a u64.")?) + common::celestia::current_block_height("https://rpc-mocha.pops.one").await } - pub async fn get_auth_token(&self) -> Result { - // celestia light auth admin --p2p.network mocha - let auth_token = - run_command("celestia", &["light", "auth", "admin", "--p2p.network", "mocha"]) - .await? - .trim() - .to_string(); - - Ok(auth_token) + async fn get_auth_token( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result { + common::celestia::get_auth_token(dot_movement, config, "mocha").await } pub async fn setup_celestia( @@ -89,16 +41,15 @@ impl Mocha { let mut config = common::celestia::make_dirs(dot_movement.clone(), config).await?; // celestia light init --p2p.network mocha - self.celestia_light_init().await?; - - // get the mocha 11 address - let address = self.get_mocha_11_address().await?; - config.appd.celestia_validator_address.replace(address.clone()); + self.celestia_light_init(&dot_movement, &config).await?; // get the auth token - let auth_token = self.get_auth_token().await?; + let auth_token = self.get_auth_token(&dot_movement, &config).await?; config.appd.celestia_auth_token.replace(auth_token.clone()); + // get the initial block height + config.initial_height = self.get_da_block_height().await?; + Ok(config) } diff --git a/protocol-units/da/movement/protocol/util/Cargo.toml b/protocol-units/da/movement/protocol/util/Cargo.toml index bdd22513a..faba32c7c 100644 --- a/protocol-units/da/movement/protocol/util/Cargo.toml +++ b/protocol-units/da/movement/protocol/util/Cargo.toml @@ -24,12 +24,12 @@ movement-da-light-node-proto = { workspace = true, features = [] } celestia-rpc = { workspace = true } celestia-types = { workspace = true } anyhow = { workspace = true } +base64 = { workspace = true } hex = { workspace = true } async-stream = { workspace = true } serde_json = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } -jsonrpsee = { workspace = true } dot-movement = { workspace = true } toml = { workspace = true } memseq-util = { workspace = true } diff --git a/protocol-units/da/movement/protocol/util/src/bin/wait_for_light_node.rs b/protocol-units/da/movement/protocol/util/src/bin/wait_for_light_node.rs index 42ff79cd2..61536e4ab 100644 --- a/protocol-units/da/movement/protocol/util/src/bin/wait_for_light_node.rs +++ b/protocol-units/da/movement/protocol/util/src/bin/wait_for_light_node.rs @@ -15,7 +15,7 @@ async fn main() -> Result<(), anyhow::Error> { // get the config file let dot_movement = dot_movement::DotMovement::try_from_env()?; - let mut config_file = dot_movement.try_get_or_create_config_file().await?; + let config_file = dot_movement.try_get_or_create_config_file().await?; // get a matching godfig object let godfig: Godfig = diff --git a/protocol-units/da/movement/protocol/util/src/blob/ir/blob.rs b/protocol-units/da/movement/protocol/util/src/blob/ir/blob.rs index fd81d9cec..d559e9c32 100644 --- a/protocol-units/da/movement/protocol/util/src/blob/ir/blob.rs +++ b/protocol-units/da/movement/protocol/util/src/blob/ir/blob.rs @@ -13,17 +13,17 @@ pub struct InnerSignedBlobV1 where C: Curve, { - pub data: InnerSignedBlobV1Data, - pub signature: Vec, - pub signer: Vec, - pub id: Id, + data: InnerSignedBlobV1Data, + signature: Vec, + signer: Vec, + id: Id, } impl InnerSignedBlobV1 where C: Curve + Verify + Digester, { - pub fn new( + pub(crate) fn new( data: InnerSignedBlobV1Data, signature: Vec, signer: Vec, @@ -165,29 +165,6 @@ where } } -#[cfg(test)] -pub mod test { - - use super::*; - use movement_da_light_node_signer::Signer; - use movement_signer::cryptography::secp256k1::Secp256k1; - use movement_signer_local::signer::LocalSigner; - - #[tokio::test] - async fn test_cannot_change_id_and_verify() -> Result<(), anyhow::Error> { - let blob = InnerSignedBlobV1Data::new(vec![1, 2, 3], 123); - let signer = Signer::new(LocalSigner::::random()); - let signed_blob = blob.try_to_sign(&signer).await?; - - let mut changed_blob = signed_blob.clone(); - changed_blob.id = Id::new(vec![1, 2, 3, 4]); - - assert!(changed_blob.try_verify().is_err()); - - Ok(()) - } -} - pub mod stream_read_response { use movement_da_light_node_proto::*; diff --git a/protocol-units/da/movement/protocol/util/src/config/local/appd.rs b/protocol-units/da/movement/protocol/util/src/config/appd.rs similarity index 74% rename from protocol-units/da/movement/protocol/util/src/config/local/appd.rs rename to protocol-units/da/movement/protocol/util/src/config/appd.rs index 2b61d5983..766872d32 100644 --- a/protocol-units/da/movement/protocol/util/src/config/local/appd.rs +++ b/protocol-units/da/movement/protocol/util/src/config/appd.rs @@ -1,8 +1,9 @@ -use crate::config::common::{ +use crate::config::default::{ default_celestia_appd_replace_args, default_celestia_appd_use_replace_args, default_celestia_chain_id, default_celestia_namespace, default_celestia_rpc_listen_hostname, default_celestia_rpc_listen_port, default_celestia_websocket_connection_hostname, - default_celestia_websocket_connection_port, default_celestia_websocket_connection_protocol, + default_celestia_websocket_connection_path, default_celestia_websocket_connection_port, + default_celestia_websocket_connection_protocol, }; use celestia_types::nmt::Namespace; @@ -31,6 +32,10 @@ pub struct Config { #[serde(default = "default_celestia_websocket_connection_port")] pub celestia_websocket_connection_port: u16, + /// The path of the Celestia Node websocket + #[serde(default = "default_celestia_websocket_connection_path")] + pub celestia_websocket_connection_path: String, + /// The auth token for the Celestia node pub celestia_auth_token: Option, @@ -69,6 +74,7 @@ impl Default for Config { celestia_websocket_connection_hostname: default_celestia_websocket_connection_hostname( ), celestia_websocket_connection_port: default_celestia_websocket_connection_port(), + celestia_websocket_connection_path: default_celestia_websocket_connection_path(), celestia_chain_id: default_celestia_chain_id(), celestia_auth_token: None, celestia_namespace: default_celestia_namespace(), @@ -79,3 +85,19 @@ impl Default for Config { } } } + +impl Config { + // FIXME: use a single URL field as the source. The format was introduced by + // Sir Tim Berners-Lee in 1994 so you don't have to compose this from + // values that need to be set in three different environment variables. + // NOTE: originally, this was not done because there was a need to reuse some parts of the URL but substitute others when running in environments like Docker. + pub fn celestia_websocket_url(&self) -> String { + format!( + "{}://{}:{}{}", + self.celestia_websocket_connection_protocol, + self.celestia_websocket_connection_hostname, + self.celestia_websocket_connection_port, + self.celestia_websocket_connection_path + ) + } +} diff --git a/protocol-units/da/movement/protocol/util/src/config/local/bridge.rs b/protocol-units/da/movement/protocol/util/src/config/bridge.rs similarity index 98% rename from protocol-units/da/movement/protocol/util/src/config/local/bridge.rs rename to protocol-units/da/movement/protocol/util/src/config/bridge.rs index 4f5a1bd39..c159d39c4 100644 --- a/protocol-units/da/movement/protocol/util/src/config/local/bridge.rs +++ b/protocol-units/da/movement/protocol/util/src/config/bridge.rs @@ -1,4 +1,4 @@ -use crate::config::common::{ +use crate::config::default::{ default_celestia_bridge_replace_args, default_celestia_bridge_use_replace_args, default_celestia_rpc_connection_hostname, default_celestia_rpc_connection_port, default_celestia_rpc_connection_protocol, default_celestia_websocket_listen_hostname, diff --git a/protocol-units/da/movement/protocol/util/src/config/local/da_light_node.rs b/protocol-units/da/movement/protocol/util/src/config/da_light_node.rs similarity index 94% rename from protocol-units/da/movement/protocol/util/src/config/local/da_light_node.rs rename to protocol-units/da/movement/protocol/util/src/config/da_light_node.rs index 0699daf1f..f95b6b68e 100644 --- a/protocol-units/da/movement/protocol/util/src/config/local/da_light_node.rs +++ b/protocol-units/da/movement/protocol/util/src/config/da_light_node.rs @@ -1,7 +1,8 @@ -use crate::config::common::{ +use crate::config::default::{ default_celestia_rpc_connection_hostname, default_celestia_rpc_connection_port, default_celestia_rpc_connection_protocol, default_celestia_websocket_connection_hostname, - default_celestia_websocket_connection_port, default_movement_da_light_node_connection_hostname, + default_celestia_websocket_connection_path, default_celestia_websocket_connection_port, + default_movement_da_light_node_connection_hostname, default_movement_da_light_node_connection_port, default_movement_da_light_node_http1, default_movement_da_light_node_listen_hostname, default_movement_da_light_node_listen_port, }; @@ -130,6 +131,10 @@ pub struct Config { #[serde(default = "default_celestia_websocket_connection_port")] pub celestia_websocket_connection_port: u16, + /// The path of the Celestia Node websocket + #[serde(default = "default_celestia_websocket_connection_path")] + pub celestia_websocket_connection_path: String, + // FIXME: disentangle listen config for the light node service // from the connection config to connect to the same service? /// The hostname to listen on for the movement-celestia-da-light-node service @@ -171,6 +176,7 @@ impl Default for Config { celestia_websocket_connection_hostname: default_celestia_websocket_connection_hostname( ), celestia_websocket_connection_port: default_celestia_websocket_connection_port(), + celestia_websocket_connection_path: default_celestia_websocket_connection_path(), movement_da_light_node_listen_hostname: default_movement_da_light_node_listen_hostname( ), movement_da_light_node_listen_port: default_movement_da_light_node_listen_port(), diff --git a/protocol-units/da/movement/protocol/util/src/config/common.rs b/protocol-units/da/movement/protocol/util/src/config/default.rs similarity index 79% rename from protocol-units/da/movement/protocol/util/src/config/common.rs rename to protocol-units/da/movement/protocol/util/src/config/default.rs index 8ec56bbc5..b443e8594 100644 --- a/protocol-units/da/movement/protocol/util/src/config/common.rs +++ b/protocol-units/da/movement/protocol/util/src/config/default.rs @@ -1,6 +1,10 @@ +use base64::prelude::*; use celestia_types::nmt::Namespace; use godfig::env_default; +use std::env; +use std::path::PathBuf; + // The default hostname for the Celestia RPC env_default!( default_celestia_rpc_listen_hostname, @@ -66,6 +70,14 @@ env_default!( 26658 ); +// The default Celestia Node websocket path +env_default!( + default_celestia_websocket_connection_path, + "CELESTIA_WEBSOCKET_CONNECTION_PATH", + String, + "".to_string() +); + // The default M1 DA Light Node listen hostname env_default!( default_movement_da_light_node_listen_hostname, @@ -108,14 +120,20 @@ env_default!( // The default Celestia Namespace pub fn default_celestia_namespace() -> Namespace { - match std::env::var("CELESTIA_NAMESPACE") { - Ok(val) => match serde_json::from_str(&val) { - Ok(namespace) => namespace, - // todo: get rid of this unwrap somehow, even though it should never fail - Err(_) => Namespace::new_v0(b"movement").unwrap(), - }, + match env::var("CELESTIA_NAMESPACE") { + Ok(val) => { + if val.starts_with("0x") { + let id = hex::decode(&val[2..]).expect("failed to decode hexadecimal namespace ID"); + Namespace::new_v0(&id).expect("invalid namespace ID") + } else { + let id = + BASE64_STANDARD.decode(&val).expect("failed to decode base64 namespace ID"); + Namespace::from_raw(&id).expect("invalid namespace ID") + } + } // todo: get rid of this unwrap somehow, even though it should never fail - Err(_) => Namespace::new_v0(b"movement").unwrap(), + Err(env::VarError::NotPresent) => Namespace::new_v0(b"movement").unwrap(), + Err(e) => panic!("invalid environment value of CELESTIA_NAMESPACE: {e}"), } } @@ -157,3 +175,15 @@ env_default!(default_da_light_node_is_initial, "MOVEMENT_DA_LIGHT_NODE_IS_INITIA // Whether to use http1 for Movement Light Node Connections env_default!(default_movement_da_light_node_http1, "MOVEMENT_DA_LIGHT_NODE_HTTP1", bool, false); + +// The Celestia light node's signing keyname +env_default!( + default_celestia_light_node_key_name, + "CELESTIA_LIGHT_KEYNAME", + String, + "movement_celestia_light".into() +); + +pub fn default_celestia_light_node_store() -> Option { + std::env::var_os("CELESTIA_LIGHT_NODE_STORE").map(Into::into) +} diff --git a/protocol-units/da/movement/protocol/util/src/config/local/digest_store.rs b/protocol-units/da/movement/protocol/util/src/config/digest_store.rs similarity index 100% rename from protocol-units/da/movement/protocol/util/src/config/local/digest_store.rs rename to protocol-units/da/movement/protocol/util/src/config/digest_store.rs diff --git a/protocol-units/da/movement/protocol/util/src/config/light.rs b/protocol-units/da/movement/protocol/util/src/config/light.rs new file mode 100644 index 000000000..da9bc2a74 --- /dev/null +++ b/protocol-units/da/movement/protocol/util/src/config/light.rs @@ -0,0 +1,24 @@ +use super::default::{default_celestia_light_node_key_name, default_celestia_light_node_store}; + +use serde::{Deserialize, Serialize}; + +use std::path::PathBuf; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Config { + /// Name of the node's signing key in the keyring + #[serde(default = "default_celestia_light_node_key_name")] + pub key_name: String, + /// Path name of the node store directory + #[serde(default)] + pub node_store: Option, +} + +impl Default for Config { + fn default() -> Self { + Self { + key_name: default_celestia_light_node_key_name(), + node_store: default_celestia_light_node_store(), + } + } +} diff --git a/protocol-units/da/movement/protocol/util/src/config/local/mod.rs b/protocol-units/da/movement/protocol/util/src/config/local/mod.rs deleted file mode 100644 index 6b09006ab..000000000 --- a/protocol-units/da/movement/protocol/util/src/config/local/mod.rs +++ /dev/null @@ -1,57 +0,0 @@ -pub mod appd; -pub mod bridge; -pub mod da_light_node; -pub mod digest_store; -use crate::config::common::{default_celestia_force_new_chain, default_da_light_node_is_initial}; -use aptos_account_whitelist::config::Config as WhitelistConfig; -use memseq_util::Config as MemseqConfig; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Config { - /// The appd configuration - #[serde(default)] - pub appd: appd::Config, - - /// The bridge configuration - #[serde(default)] - pub bridge: bridge::Config, - - /// The movement-celestia-da-light-node configuration - #[serde(default)] - pub da_light_node: da_light_node::Config, - - /// Whether to force a new chain - #[serde(default = "default_celestia_force_new_chain")] - pub celestia_force_new_chain: bool, - - /// The memseq configuration - #[serde(default)] - pub memseq: MemseqConfig, - - #[serde(default = "default_da_light_node_is_initial")] - pub da_light_node_is_initial: bool, - - /// The access control config - #[serde(default)] - pub access_control: WhitelistConfig, - - /// The digest store configuration - #[serde(default)] - pub digest_store: digest_store::Config, -} - -impl Default for Config { - fn default() -> Self { - Self { - appd: appd::Config::default(), - bridge: bridge::Config::default(), - da_light_node: da_light_node::Config::default(), - celestia_force_new_chain: default_celestia_force_new_chain(), - memseq: MemseqConfig::default(), - da_light_node_is_initial: default_da_light_node_is_initial(), - access_control: WhitelistConfig::default(), - digest_store: digest_store::Config::default(), - } - } -} diff --git a/protocol-units/da/movement/protocol/util/src/config/mod.rs b/protocol-units/da/movement/protocol/util/src/config/mod.rs index e6c35b1ff..81936fd10 100644 --- a/protocol-units/da/movement/protocol/util/src/config/mod.rs +++ b/protocol-units/da/movement/protocol/util/src/config/mod.rs @@ -1,7 +1,18 @@ +pub mod appd; +pub mod bridge; +pub mod da_light_node; +pub mod default; +pub mod digest_store; +pub mod light; + +use self::default::{default_celestia_force_new_chain, default_da_light_node_is_initial}; + use anyhow::Context; +use aptos_account_whitelist::config::Config as WhitelistConfig; use aptos_types::account_address::AccountAddress; use celestia_rpc::Client; use celestia_types::nmt::Namespace; +use memseq_util::Config as MemseqConfig; use movement_signer::cryptography::{secp256k1::Secp256k1, Curve}; use movement_signer_loader::{Load, LoadedSigner}; use serde::{Deserialize, Serialize}; @@ -9,153 +20,127 @@ use std::collections::HashSet; use std::future::Future; use std::path::PathBuf; -pub mod common; -pub mod local; +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum Network { + Local, + Arabica, + Mocha, + Mainnet, +} #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum Config { - Local(local::Config), - Arabica(local::Config), - Mocha(local::Config), +pub struct Config { + pub network: Network, + + /// The appd configuration + #[serde(default)] + pub appd: appd::Config, + + /// The bridge configuration + #[serde(default)] + pub bridge: bridge::Config, + + /// The celestia light node configuration. + /// Not to be confused by the movement-celestia-da-light-node-configuration. + #[serde(default)] + pub light: light::Config, + + /// The movement-celestia-da-light-node configuration + #[serde(default)] + pub da_light_node: da_light_node::Config, + + /// Whether to force a new chain + #[serde(default = "default_celestia_force_new_chain")] + pub celestia_force_new_chain: bool, + + /// The memseq configuration + #[serde(default)] + pub memseq: MemseqConfig, + + #[serde(default = "default_da_light_node_is_initial")] + pub da_light_node_is_initial: bool, + + /// The height to begin fetching blobs from. Filled by querying the network + /// during the initial setup. + #[serde(default)] + pub initial_height: u64, + + /// The access control config + #[serde(default)] + pub access_control: WhitelistConfig, + + /// The digest store configuration + #[serde(default)] + pub digest_store: digest_store::Config, } impl Default for Config { fn default() -> Self { - std::env::var("CELESTIA_NETWORK").map_or_else( - |_| Config::Local(local::Config::default()), + let network = std::env::var("CELESTIA_NETWORK").map_or_else( + |_| Network::Local, |network| match network.as_str() { - "arabica" => Config::Arabica(local::Config::default()), - "mocha" => Config::Mocha(local::Config::default()), - _ => Config::Local(local::Config::default()), + "mainnet" => Network::Mainnet, + "arabica" => Network::Arabica, + "mocha" => Network::Mocha, + _ => Network::Local, }, - ) + ); + let celestia_force_new_chain = + if network == Network::Local { default_celestia_force_new_chain() } else { false }; + Self { + network, + appd: appd::Config::default(), + bridge: bridge::Config::default(), + light: light::Config::default(), + da_light_node: da_light_node::Config::default(), + celestia_force_new_chain, + memseq: MemseqConfig::default(), + da_light_node_is_initial: default_da_light_node_is_initial(), + initial_height: 0, + access_control: WhitelistConfig::default(), + digest_store: digest_store::Config::default(), + } } } impl Config { /// Connects to a Celestia node using the config pub async fn connect_celestia(&self) -> Result { - match self { - Config::Local(local) => { - let celestia_node_url = format!( - "{}://{}:{}", - local.appd.celestia_websocket_connection_protocol, - local.appd.celestia_websocket_connection_hostname, - local.appd.celestia_websocket_connection_port - ); - let celestia_auth_token = local.appd.celestia_auth_token.clone().context( - "Failed to get Celestia auth token from config. This is required for connecting to Celestia.", - )?; - - let client = Client::new(&celestia_node_url, Some(&celestia_auth_token)) - .await - .map_err(|e| { - anyhow::anyhow!( - "Failed to connect to Celestia client at {:?}: {}", - celestia_node_url, - e - ) - })?; - - Ok(client) - } - Config::Arabica(local) => { - // arabica is also local for now - let celestia_node_url = format!( - "{}://{}:{}", - local.appd.celestia_websocket_connection_protocol, - local.appd.celestia_websocket_connection_hostname, - local.appd.celestia_websocket_connection_port - ); - let celestia_auth_token = local.appd.celestia_auth_token.clone().context( - "Failed to get Celestia auth token from config. This is required for connecting to Celestia.", - )?; - - let client = Client::new(&celestia_node_url, Some(&celestia_auth_token)) - .await - .map_err(|e| { - anyhow::anyhow!( - "Failed to connect to Celestia client at {:?}: {}", - celestia_node_url, - e - ) - })?; - - Ok(client) - } - Config::Mocha(local) => { - // mocha is also local for now - let celestia_node_url = format!( - "{}://{}:{}", - local.appd.celestia_websocket_connection_protocol, - local.appd.celestia_websocket_connection_hostname, - local.appd.celestia_websocket_connection_port - ); - let celestia_auth_token = local.appd.celestia_auth_token.clone().context( - "Failed to get Celestia auth token from config. This is required for connecting to Celestia.", - )?; - - let client = Client::new(&celestia_node_url, Some(&celestia_auth_token)) - .await - .map_err(|e| { - anyhow::anyhow!( - "Failed to connect to Celestia client at {:?}: {}", - celestia_node_url, - e - ) - })?; - - Ok(client) - } - } + let celestia_node_url = self.appd.celestia_websocket_url(); + let celestia_auth_token = self.appd.celestia_auth_token.clone().context( + "Failed to get Celestia auth token from config. This is required for connecting to Celestia.", + )?; + + let client = + Client::new(&celestia_node_url, Some(&celestia_auth_token)).await.map_err(|e| { + anyhow::anyhow!( + "Failed to connect to Celestia client at {:?}: {}", + celestia_node_url, + e + ) + })?; + + Ok(client) } /// Gets the Celestia namespace pub fn celestia_namespace(&self) -> Namespace { - match self { - Config::Local(local) => local.appd.celestia_namespace.clone(), - Config::Arabica(local) => local.appd.celestia_namespace.clone(), - Config::Mocha(local) => local.appd.celestia_namespace.clone(), - } + self.appd.celestia_namespace.clone() } /// Gets M1 DA Light Node connection protocol pub fn movement_da_light_node_connection_protocol(&self) -> String { - match self { - Config::Local(local) => { - local.da_light_node.movement_da_light_node_connection_protocol.clone() - } - Config::Arabica(local) => { - local.da_light_node.movement_da_light_node_connection_protocol.clone() - } - Config::Mocha(local) => { - local.da_light_node.movement_da_light_node_connection_protocol.clone() - } - } + self.da_light_node.movement_da_light_node_connection_protocol.clone() } /// Gets M1 DA Light Node listen hostname pub fn movement_da_light_node_listen_hostname(&self) -> String { - match self { - Config::Local(local) => { - local.da_light_node.movement_da_light_node_listen_hostname.clone() - } - Config::Arabica(local) => { - local.da_light_node.movement_da_light_node_listen_hostname.clone() - } - Config::Mocha(local) => { - local.da_light_node.movement_da_light_node_listen_hostname.clone() - } - } + self.da_light_node.movement_da_light_node_listen_hostname.clone() } /// Gets M1 DA Light Node listen port pub fn movement_da_light_node_listen_port(&self) -> u16 { - match self { - Config::Local(local) => local.da_light_node.movement_da_light_node_listen_port, - Config::Arabica(local) => local.da_light_node.movement_da_light_node_listen_port, - Config::Mocha(local) => local.da_light_node.movement_da_light_node_listen_port, - } + self.da_light_node.movement_da_light_node_listen_port } /// Gets M1 DA Light Node service @@ -167,89 +152,41 @@ impl Config { /// Gets M1 DA Light Node connection hostname pub fn movement_da_light_node_connection_hostname(&self) -> String { - match self { - Config::Local(local) => { - local.da_light_node.movement_da_light_node_connection_hostname.clone() - } - Config::Arabica(local) => { - local.da_light_node.movement_da_light_node_connection_hostname.clone() - } - Config::Mocha(local) => { - local.da_light_node.movement_da_light_node_connection_hostname.clone() - } - } + self.da_light_node.movement_da_light_node_connection_hostname.clone() } /// Gets M1 DA Light Node connection port pub fn movement_da_light_node_connection_port(&self) -> u16 { - match self { - Config::Local(local) => local.da_light_node.movement_da_light_node_connection_port, - Config::Arabica(local) => local.da_light_node.movement_da_light_node_connection_port, - Config::Mocha(local) => local.da_light_node.movement_da_light_node_connection_port, - } + self.da_light_node.movement_da_light_node_connection_port } /// Whether to use HTTP/1.1 for the movement-da-light-node service pub fn movement_da_light_node_http1(&self) -> bool { - match self { - Config::Local(local) => local.da_light_node.movement_da_light_node_http1, - Config::Arabica(local) => local.da_light_node.movement_da_light_node_http1, - Config::Mocha(local) => local.da_light_node.movement_da_light_node_http1, - } + self.da_light_node.movement_da_light_node_http1 } /// Gets the memseq path pub fn try_memseq_path(&self) -> Result { - match self { - Config::Local(local) => local.memseq.sequencer_database_path.clone().context( + self.memseq.sequencer_database_path.clone().context( "Failed to get memseq path from config. This is required for initializing the memseq database.", - ), - Config::Arabica(local) => local.memseq.sequencer_database_path.clone().context( - "Failed to get memseq path from config. This is required for initializing the memseq database.", - ), - Config::Mocha(local) => local.memseq.sequencer_database_path.clone().context( - "Failed to get memseq path from config. This is required for initializing the memseq database.", - ), - } + ) } /// Gets the da signers sec1 keys pub fn da_signers_sec1_keys(&self) -> HashSet { - match self { - Config::Local(local) => local.da_light_node.da_signers.public_keys_hex.clone(), - Config::Arabica(local) => local.da_light_node.da_signers.public_keys_hex.clone(), - Config::Mocha(local) => local.da_light_node.da_signers.public_keys_hex.clone(), - } + self.da_light_node.da_signers.public_keys_hex.clone() } - pub fn try_block_building_parameters(&self) -> Result<(u32, u64), anyhow::Error> { - match self { - Config::Local(local) => { - Ok((local.memseq.memseq_max_block_size, local.memseq.memseq_build_time)) - } - Config::Arabica(local) => { - Ok((local.memseq.memseq_max_block_size, local.memseq.memseq_build_time)) - } - Config::Mocha(local) => { - Ok((local.memseq.memseq_max_block_size, local.memseq.memseq_build_time)) - } - } + pub fn block_building_parameters(&self) -> (u32, u64) { + (self.memseq.memseq_max_block_size, self.memseq.memseq_build_time) } pub fn whitelisted_accounts(&self) -> Result>, anyhow::Error> { - match self { - Config::Local(local) => local.access_control.whitelisted_accounts(), - Config::Arabica(local) => local.access_control.whitelisted_accounts(), - Config::Mocha(local) => local.access_control.whitelisted_accounts(), - } + self.access_control.whitelisted_accounts() } pub fn digest_store_db_path(&self) -> PathBuf { - match self { - Config::Local(local) => local.digest_store.digest_store_db_path.clone(), - Config::Arabica(local) => local.digest_store.digest_store_db_path.clone(), - Config::Mocha(local) => local.digest_store.digest_store_db_path.clone(), - } + self.digest_store.digest_store_db_path.clone() } } @@ -263,35 +200,13 @@ where impl LoadSigner for Config { async fn da_signer(&self) -> Result, anyhow::Error> { - match self { - Config::Local(local) => { - let identifier: Box + Send> = - Box::new(local.da_light_node.da_signers.signer_identifier.clone()); - let signer = identifier - .load() - .await - .map_err(|e| anyhow::anyhow!("failed to load signer: {}", e))?; - Ok(signer) - } - Config::Arabica(local) => { - let identifier: Box + Send> = - Box::new(local.da_light_node.da_signers.signer_identifier.clone()); - let signer = identifier - .load() - .await - .map_err(|e| anyhow::anyhow!("failed to load signer: {}", e))?; - Ok(signer) - } - Config::Mocha(local) => { - let identifier: Box + Send> = - Box::new(local.da_light_node.da_signers.signer_identifier.clone()); - let signer = identifier - .load() - .await - .map_err(|e| anyhow::anyhow!("failed to load signer: {}", e))?; - Ok(signer) - } - } + let identifier: Box + Send> = + Box::new(self.da_light_node.da_signers.signer_identifier.clone()); + let signer = identifier + .load() + .await + .map_err(|e| anyhow::anyhow!("failed to load signer: {}", e))?; + Ok(signer) } } diff --git a/protocol-units/da/movement/providers/celestia/src/da/mod.rs b/protocol-units/da/movement/providers/celestia/src/da/mod.rs index 361df7691..b14532075 100644 --- a/protocol-units/da/movement/providers/celestia/src/da/mod.rs +++ b/protocol-units/da/movement/providers/celestia/src/da/mod.rs @@ -1,14 +1,15 @@ use crate::blob::ir::{into_da_blob, CelestiaDaBlob}; -use celestia_rpc::{BlobClient, Client, HeaderClient}; -use celestia_types::{nmt::Namespace, Blob as CelestiaBlob, TxConfig}; +use celestia_rpc::{BlobClient, Client, HeaderClient, TxConfig}; +use celestia_types::{nmt::Namespace, Blob as CelestiaBlob}; use movement_da_light_node_da::{Certificate, CertificateStream, DaError, DaOperations}; use movement_da_util::blob::ir::blob::DaBlob; use movement_signer::cryptography::Curve; use serde::{Deserialize, Serialize}; +use std::fmt::Debug; use std::future::Future; use std::pin::Pin; use std::sync::Arc; -use tracing::error; +use tracing::{debug, error}; #[derive(Clone)] pub struct Da @@ -53,25 +54,22 @@ where impl DaOperations for Da where - C: Curve - + Send - + Sync - + Clone - + Serialize - + for<'de> Deserialize<'de> - + 'static - + std::fmt::Debug, + C: Curve + Send + Sync + Clone + Serialize + Debug + for<'de> Deserialize<'de> + 'static, { fn submit_blob( &self, data: DaBlob, ) -> Pin> + Send + '_>> { Box::pin(async move { + debug!("submitting blob to celestia {:?}", data); + // create the blob let celestia_blob = self .create_new_celestia_blob(data) .map_err(|e| DaError::Internal(format!("failed to create celestia blob :{e}")))?; + debug!("created celestia blob {:?}", celestia_blob); + // submit the blob to the celestia node self.submit_celestia_blob(celestia_blob) .await @@ -86,6 +84,7 @@ where height: u64, ) -> Pin>, DaError>> + Send + '_>> { Box::pin(async move { + debug!("getting blobs at height {height}"); let height = if height == 0 { 1 } else { height }; match self.default_client.blob_get_all(height, &[self.celestia_namespace]).await { @@ -106,6 +105,7 @@ where format!("failed to convert blob: {e}").into(), ) })?; + debug!("got blob {da_blob:?}"); da_blobs.push(da_blob); } diff --git a/protocol-units/execution/maptos/dof/src/v1.rs b/protocol-units/execution/maptos/dof/src/v1.rs index b8d92b63f..c3779eb68 100644 --- a/protocol-units/execution/maptos/dof/src/v1.rs +++ b/protocol-units/execution/maptos/dof/src/v1.rs @@ -32,8 +32,8 @@ impl Executor { Self { executor, finality_view } } - pub fn try_from_config(config: Config) -> Result { - let executor = OptExecutor::try_from_config(config)?; + pub async fn try_from_config(config: Config) -> Result { + let executor = OptExecutor::try_from_config(config).await?; Ok(Self::new(executor)) } } @@ -174,11 +174,11 @@ mod tests { use std::collections::HashMap; - fn setup(mut maptos_config: Config) -> Result<(Executor, TempDir), anyhow::Error> { + async fn setup(mut maptos_config: Config) -> Result<(Executor, TempDir), anyhow::Error> { let tempdir = tempfile::tempdir()?; // replace the db path with the temporary directory maptos_config.chain.maptos_db_path.replace(tempdir.path().to_path_buf()); - let executor = Executor::try_from_config(maptos_config)?; + let executor = Executor::try_from_config(maptos_config).await?; Ok((executor, tempdir)) } @@ -205,7 +205,7 @@ mod tests { config.chain.maptos_private_key_signer_identifier = SignerIdentifier::Local(Local { private_key_hex_bytes: private_key.to_encoded_string()?.to_string(), }); - let (executor, _tempdir) = setup(config)?; + let (executor, _tempdir) = setup(config).await?; let block_id = HashValue::random(); let block_metadata = executor .build_block_metadata(block_id.clone(), chrono::Utc::now().timestamp_micros() as u64) @@ -231,7 +231,7 @@ mod tests { config.chain.maptos_private_key_signer_identifier = SignerIdentifier::Local(Local { private_key_hex_bytes: private_key.to_encoded_string()?.to_string(), }); - let (executor, _tempdir) = setup(config.clone())?; + let (executor, _tempdir) = setup(config.clone()).await?; let (tx_sender, mut tx_receiver) = mpsc::channel(16); let (context, background) = executor.background(tx_sender, &config)?; let services = context.services(); @@ -265,7 +265,7 @@ mod tests { }); config.chain.maptos_read_only = true; let (tx_sender, _tx_receiver) = mpsc::channel(16); - let (executor, _tempdir) = setup(config.clone())?; + let (executor, _tempdir) = setup(config.clone()).await?; let (context, background) = executor.background(tx_sender, &config)?; let services = context.services(); let api = services.get_opt_apis(); @@ -295,7 +295,7 @@ mod tests { config.chain.maptos_private_key_signer_identifier = SignerIdentifier::Local(Local { private_key_hex_bytes: private_key.to_encoded_string()?.to_string(), }); - let (executor, _tempdir) = setup(config.clone())?; + let (executor, _tempdir) = setup(config.clone()).await?; let (tx_sender, mut tx_receiver) = mpsc::channel(16); let (context, background) = executor.background(tx_sender, &config)?; let services = context.services(); @@ -356,7 +356,7 @@ mod tests { config.chain.maptos_private_key_signer_identifier = SignerIdentifier::Local(Local { private_key_hex_bytes: private_key.to_encoded_string()?.to_string(), }); - let (executor, _tempdir) = setup(config.clone())?; + let (executor, _tempdir) = setup(config.clone()).await?; let (tx_sender, mut tx_receiver) = mpsc::channel(16); let (context, background) = executor.background(tx_sender, &config)?; let services = context.services(); @@ -438,7 +438,7 @@ mod tests { // Create an executor instance from the environment configuration. let config = Config::default(); let (tx_sender, _tx_receiver) = mpsc::channel(16); - let executor = Executor::try_from_config(config.clone())?; + let executor = Executor::try_from_config(config.clone()).await?; let (context, background) = executor.background(tx_sender, &config)?; let config = executor.config(); let services = context.services(); @@ -511,7 +511,7 @@ mod tests { // Create an executor instance from the environment configuration. let config = Config::default(); let (tx_sender, _tx_receiver) = mpsc::channel(16); - let executor = Executor::try_from_config(config.clone())?; + let executor = Executor::try_from_config(config.clone()).await?; let (context, background) = executor.background(tx_sender, &config)?; let config = executor.config(); let services = context.services(); diff --git a/protocol-units/execution/maptos/fin-view/src/fin_view.rs b/protocol-units/execution/maptos/fin-view/src/fin_view.rs index 7afaacff1..97570c01a 100644 --- a/protocol-units/execution/maptos/fin-view/src/fin_view.rs +++ b/protocol-units/execution/maptos/fin-view/src/fin_view.rs @@ -79,7 +79,7 @@ mod tests { // Create an Executor and a FinalityView instance from the environment configuration. let config = Config::default(); let (tx_sender, _tx_receiver) = mpsc::channel(16); - let executor = Executor::try_from_config(config)?; + let executor = Executor::try_from_config(config).await?; let (context, _transaction_pipe) = executor.background(tx_sender)?; let finality_view = FinalityView::new(context.db_reader()); let service = finality_view.service( diff --git a/protocol-units/execution/maptos/framework/releases/biarritz-rc1/mrb_cache/aa45303216be96ea30d361ab7eb2e95fb08c2dcb-biarritz-rc1.mrb b/protocol-units/execution/maptos/framework/releases/biarritz-rc1/mrb_cache/aa45303216be96ea30d361ab7eb2e95fb08c2dcb-biarritz-rc1.mrb new file mode 100644 index 000000000..51cc61e95 Binary files /dev/null and b/protocol-units/execution/maptos/framework/releases/biarritz-rc1/mrb_cache/aa45303216be96ea30d361ab7eb2e95fb08c2dcb-biarritz-rc1.mrb differ diff --git a/protocol-units/execution/maptos/opt-executor/Cargo.toml b/protocol-units/execution/maptos/opt-executor/Cargo.toml index 59177a71f..e076854ce 100644 --- a/protocol-units/execution/maptos/opt-executor/Cargo.toml +++ b/protocol-units/execution/maptos/opt-executor/Cargo.toml @@ -76,6 +76,7 @@ aptos-gas-schedule = { workspace = true } aptos-sdk = { workspace = true } tempfile = { workspace = true } movement-signer-loader = { workspace = true } +movement-signer = { workspace = true } [dev-dependencies] dirs = { workspace = true } diff --git a/protocol-units/execution/maptos/opt-executor/src/background/transaction_pipe.rs b/protocol-units/execution/maptos/opt-executor/src/background/transaction_pipe.rs index 875377486..415abf4d4 100644 --- a/protocol-units/execution/maptos/opt-executor/src/background/transaction_pipe.rs +++ b/protocol-units/execution/maptos/opt-executor/src/background/transaction_pipe.rs @@ -344,9 +344,12 @@ mod tests { use maptos_execution_util::config::chain::Config; use tempfile::TempDir; - fn setup() -> (Context, TransactionPipe, mpsc::Receiver<(u64, SignedTransaction)>, TempDir) { + async fn setup() -> (Context, TransactionPipe, mpsc::Receiver<(u64, SignedTransaction)>, TempDir) + { let (tx_sender, tx_receiver) = mpsc::channel(16); - let (executor, tempdir) = Executor::try_test_default(GENESIS_KEYPAIR.0.clone()).unwrap(); + let (executor, tempdir) = Executor::try_test_default(GENESIS_KEYPAIR.0.clone()) + .await + .expect("Transaction pipe test can't create executor with private key."); let (context, background) = executor.background(tx_sender).unwrap(); let transaction_pipe = background.into_transaction_pipe(); (context, transaction_pipe, tx_receiver, tempdir) @@ -367,7 +370,7 @@ mod tests { async fn test_pipe_mempool() -> Result<(), anyhow::Error> { // set up let maptos_config = Config::default(); - let (context, mut transaction_pipe, mut tx_receiver, _tempdir) = setup(); + let (context, mut transaction_pipe, mut tx_receiver, _tempdir) = setup().await; let user_transaction = create_signed_transaction(1, &maptos_config); // send transaction to mempool @@ -395,7 +398,7 @@ mod tests { async fn test_pipe_mempool_cancellation() -> Result<(), anyhow::Error> { // set up let maptos_config = Config::default(); - let (context, mut transaction_pipe, _tx_receiver, _tempdir) = setup(); + let (context, mut transaction_pipe, _tx_receiver, _tempdir) = setup().await; let user_transaction = create_signed_transaction(1, &maptos_config); // send transaction to mempool @@ -418,7 +421,7 @@ mod tests { async fn test_pipe_mempool_with_duplicate_transaction() -> Result<(), anyhow::Error> { // set up let maptos_config = Config::default(); - let (context, mut transaction_pipe, mut tx_receiver, _tempdir) = setup(); + let (context, mut transaction_pipe, mut tx_receiver, _tempdir) = setup().await; let mut mempool_client_sender = context.mempool_client_sender(); let user_transaction = create_signed_transaction(1, &maptos_config); @@ -459,7 +462,7 @@ mod tests { #[tokio::test] async fn test_pipe_mempool_from_api() -> Result<(), anyhow::Error> { - let (context, mut transaction_pipe, mut tx_receiver, _tempdir) = setup(); + let (context, mut transaction_pipe, mut tx_receiver, _tempdir) = setup().await; let service = Service::new(&context); #[allow(unreachable_code)] @@ -486,7 +489,7 @@ mod tests { #[tokio::test] async fn test_repeated_pipe_mempool_from_api() -> Result<(), anyhow::Error> { - let (context, mut transaction_pipe, mut tx_receiver, _tempdir) = setup(); + let (context, mut transaction_pipe, mut tx_receiver, _tempdir) = setup().await; let service = Service::new(&context); #[allow(unreachable_code)] @@ -526,7 +529,7 @@ mod tests { async fn test_cannot_submit_too_new() -> Result<(), anyhow::Error> { // set up let maptos_config = Config::default(); - let (_context, mut transaction_pipe, _tx_receiver, _tempdir) = setup(); + let (_context, mut transaction_pipe, _tx_receiver, _tempdir) = setup().await; // submit a transaction with a valid sequence number let user_transaction = create_signed_transaction(0, &maptos_config); @@ -554,7 +557,7 @@ mod tests { #[tokio::test] async fn test_sequence_number_too_old() -> Result<(), anyhow::Error> { let (tx_sender, _tx_receiver) = mpsc::channel(16); - let (executor, _tempdir) = Executor::try_test_default(GENESIS_KEYPAIR.0.clone())?; + let (executor, _tempdir) = Executor::try_test_default(GENESIS_KEYPAIR.0.clone()).await?; let (context, background) = executor.background(tx_sender)?; let mut transaction_pipe = background.into_transaction_pipe(); diff --git a/protocol-units/execution/maptos/opt-executor/src/bootstrap.rs b/protocol-units/execution/maptos/opt-executor/src/bootstrap.rs index a1b7685d4..de08a988c 100644 --- a/protocol-units/execution/maptos/opt-executor/src/bootstrap.rs +++ b/protocol-units/execution/maptos/opt-executor/src/bootstrap.rs @@ -4,7 +4,6 @@ use aptos_crypto::ed25519::Ed25519PublicKey; use aptos_db::AptosDB; use aptos_executor::db_bootstrapper; use aptos_gas_schedule::{AptosGasParameters, InitialGasSchedule, ToOnChainGasSchedule}; -use aptos_sdk::move_types::gas_algebra::GasQuantity; use aptos_storage_interface::DbReaderWriter; use aptos_types::{ chain_id::ChainId, diff --git a/protocol-units/execution/maptos/opt-executor/src/executor/execution.rs b/protocol-units/execution/maptos/opt-executor/src/executor/execution.rs index 77c3ea200..6a2962e41 100644 --- a/protocol-units/execution/maptos/opt-executor/src/executor/execution.rs +++ b/protocol-units/execution/maptos/opt-executor/src/executor/execution.rs @@ -212,11 +212,14 @@ mod tests { ed25519::{Ed25519PrivateKey, Ed25519Signature}, HashValue, PrivateKey, Uniform, }; - use aptos_sdk::{transaction_builder::TransactionFactory, types::LocalAccount}; + use aptos_sdk::{ + transaction_builder::TransactionFactory, + types::{AccountKey, LocalAccount}, + }; use aptos_storage_interface::state_view::DbStateViewAtVersion; use aptos_types::{ account_address::AccountAddress, - account_config::AccountResource, + account_config::{aptos_test_root_address, AccountResource}, block_executor::partitioner::ExecutableTransactions, block_metadata::BlockMetadata, chain_id::ChainId, @@ -250,7 +253,7 @@ mod tests { async fn test_execute_block() -> Result<(), anyhow::Error> { let private_key = Ed25519PrivateKey::generate_for_testing(); let (tx_sender, _tx_receiver) = mpsc::channel(1); - let (executor, _tempdir) = Executor::try_test_default(private_key)?; + let (executor, _tempdir) = Executor::try_test_default(private_key).await?; let (context, _transaction_pipe) = executor.background(tx_sender)?; let block_id = HashValue::random(); let block_metadata = Transaction::BlockMetadata(BlockMetadata::new( @@ -284,7 +287,7 @@ mod tests { // Create an executor instance from the environment configuration. let private_key = Ed25519PrivateKey::generate_for_testing(); let (tx_sender, _tx_receiver) = mpsc::channel(1); - let (executor, _tempdir) = Executor::try_test_default(private_key)?; + let (executor, _tempdir) = Executor::try_test_default(private_key).await?; let (context, _transaction_pipe) = executor.background(tx_sender)?; // Initialize a root account using a predefined keypair and the test root address. @@ -295,8 +298,11 @@ mod tests { .maptos_private_key_signer_identifier .try_raw_private_key()?; let private_key = Ed25519PrivateKey::try_from(raw_private_key.as_slice())?; - let root_account = - LocalAccount::from_private_key(private_key.to_encoded_string()?.as_str(), 0)?; + let root_account = LocalAccount::new( + aptos_test_root_address(), + AccountKey::from_private_key(private_key), + 0, + ); // Seed for random number generator, used here to generate predictable results in a test environment. let seed = [3u8; 32]; @@ -392,7 +398,7 @@ mod tests { // Create an executor instance from the environment configuration. let private_key = Ed25519PrivateKey::generate_for_testing(); let (tx_sender, _tx_receiver) = mpsc::channel(16); - let (executor, _tempdir) = Executor::try_test_default(private_key)?; + let (executor, _tempdir) = Executor::try_test_default(private_key).await?; let (context, _transaction_pipe) = executor.background(tx_sender)?; let service = Service::new(&context); @@ -403,8 +409,11 @@ mod tests { .maptos_private_key_signer_identifier .try_raw_private_key()?; let private_key = Ed25519PrivateKey::try_from(raw_private_key.as_slice())?; - let root_account = - LocalAccount::from_private_key(private_key.to_encoded_string()?.as_str(), 0)?; + let root_account = LocalAccount::new( + aptos_test_root_address(), + AccountKey::from_private_key(private_key), + 0, + ); // Seed for random number generator, used here to generate predictable results in a test environment. let seed = [3u8; 32]; diff --git a/protocol-units/execution/maptos/opt-executor/src/executor/initialization.rs b/protocol-units/execution/maptos/opt-executor/src/executor/initialization.rs index 2cf149ba6..915159add 100644 --- a/protocol-units/execution/maptos/opt-executor/src/executor/initialization.rs +++ b/protocol-units/execution/maptos/opt-executor/src/executor/initialization.rs @@ -1,11 +1,14 @@ use super::Executor; use crate::background::BackgroundTask; use crate::{bootstrap, Context}; -use aptos_crypto::ed25519::Ed25519PrivateKey; +use movement_signer::cryptography::ed25519::Ed25519; +use movement_signer::Signing; +use movement_signer_loader::{Load, LoadedSigner}; use aptos_config::config::NodeConfig; +#[cfg(test)] +use aptos_crypto::ed25519::Ed25519PrivateKey; use aptos_crypto::ed25519::Ed25519PublicKey; -use aptos_crypto::PrivateKey; #[cfg(test)] use aptos_crypto::ValidCryptoMaterialStringExt; use aptos_executor::block_executor::BlockExecutor; @@ -128,16 +131,21 @@ impl Executor { }) } - pub fn bootstrap(maptos_config: &Config) -> Result { - let raw_private_key = - maptos_config.chain.maptos_private_key_signer_identifier.try_raw_private_key()?; - let private_key = Ed25519PrivateKey::try_from(raw_private_key.as_slice())?; - let public_key = private_key.public_key(); + pub async fn bootstrap(maptos_config: &Config) -> Result { + // let raw_private_key = + // maptos_config.chain.maptos_private_key_signer_identifier.try_raw_private_key()?; + // let private_key = Ed25519PrivateKey::try_from(raw_private_key.as_slice())?; + // let public_key = private_key.public_key(); + + let loader: LoadedSigner = + maptos_config.chain.maptos_private_key_signer_identifier.load().await?; + let public_key = Ed25519PublicKey::try_from(loader.public_key().await?.as_bytes())?; + Self::bootstrap_with_public_key(maptos_config, public_key) } - pub fn try_from_config(maptos_config: Config) -> Result { - Self::bootstrap(&maptos_config) + pub async fn try_from_config(maptos_config: Config) -> Result { + Self::bootstrap(&maptos_config).await } #[cfg(test)] @@ -163,7 +171,7 @@ impl Executor { } #[cfg(test)] - pub fn try_test_default( + pub async fn try_test_default( private_key: Ed25519PrivateKey, ) -> Result<(Self, TempDir), anyhow::Error> { let tempdir = tempfile::tempdir()?; @@ -177,7 +185,7 @@ impl Executor { // replace the db path with the temporary directory maptos_config.chain.maptos_db_path.replace(tempdir.path().to_path_buf()); - let executor = Self::try_from_config(maptos_config)?; + let executor = Self::try_from_config(maptos_config).await?; Ok((executor, tempdir)) } diff --git a/protocol-units/execution/maptos/opt-executor/src/service.rs b/protocol-units/execution/maptos/opt-executor/src/service.rs index 45ed519a6..670df6877 100644 --- a/protocol-units/execution/maptos/opt-executor/src/service.rs +++ b/protocol-units/execution/maptos/opt-executor/src/service.rs @@ -115,7 +115,7 @@ mod tests { #[tokio::test] async fn test_pipe_mempool_while_server_running() -> Result<(), anyhow::Error> { let (tx_sender, mut tx_receiver) = mpsc::channel(16); - let (executor, _tempdir) = Executor::try_test_default(GENESIS_KEYPAIR.0.clone())?; + let (executor, _tempdir) = Executor::try_test_default(GENESIS_KEYPAIR.0.clone()).await?; let (context, background) = executor.background(tx_sender)?; let mut transaction_pipe = background.into_transaction_pipe(); let service = Service::new(&context); diff --git a/util/commander/Cargo.toml b/util/commander/Cargo.toml index e78460920..ded91b392 100644 --- a/util/commander/Cargo.toml +++ b/util/commander/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "commander" version = { workspace = true } -edition = { workspace = true } -license = { workspace = true } +edition = { workspace = true } +license = { workspace = true } authors = { workspace = true } repository = { workspace = true } homepage = { workspace = true } @@ -16,6 +16,7 @@ anyhow = { workspace = true } tokio = { workspace = true } futures = { workspace = true } tracing = { workspace = true } +itertools = { workspace = true } [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/util/commander/src/lib.rs b/util/commander/src/lib.rs index b5a16aaf4..885359bc8 100644 --- a/util/commander/src/lib.rs +++ b/util/commander/src/lib.rs @@ -1,10 +1,13 @@ use anyhow::Result; use futures::future::try_join; -use std::process::Stdio; +use itertools::Itertools; use tokio::io::{self, AsyncBufReadExt, AsyncWriteExt, BufReader}; -use tokio::process::Command; +use tokio::process::Command as InnerCommand; use tokio::signal::unix::{signal, SignalKind}; -use tokio::task::JoinHandle; +use tracing::info; + +use std::ffi::OsStr; +use std::process::Stdio; async fn pipe_output( reader: R, @@ -37,128 +40,110 @@ async fn pipe_error_output( } /// Runs a command, piping its output to stdout and stderr, and returns the stdout output if successful. -pub async fn run_command(command: &str, args: &[&str]) -> Result { - // print command out with args joined by space - tracing::info!("Running command: {} {}", command, args.join(" ")); - - // Setup signal handling to terminate the child process - let (tx, rx) = tokio::sync::oneshot::channel(); +pub async fn run_command(command: C, args: I) -> Result +where + C: AsRef, + I: IntoIterator, + S: AsRef, +{ + let mut command = Command::new(command); + command.args(args); + command.run_and_capture_output().await +} - let mut sigterm = signal(SignalKind::terminate())?; - let mut sigint = signal(SignalKind::interrupt())?; - let mut sigquit = signal(SignalKind::quit())?; +/// Builder for running commands +pub struct Command(InnerCommand); - tokio::spawn(async move { - tokio::select! { - _ = sigterm.recv() => { - let _ = tx.send(()); - } - _ = sigint.recv() => { - let _ = tx.send(()); - } - _ = sigquit.recv() => { - let _ = tx.send(()); - } - } - }); +impl Command { + pub fn new(program: impl AsRef) -> Self { + let inner = InnerCommand::new(program); + Self(inner) + } - let mut child = Command::new(command) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; + pub fn arg(&mut self, arg: S) -> &mut Self + where + S: AsRef, + { + self.0.arg(arg); + self + } - let stdout = child.stdout.take().ok_or_else(|| { - anyhow::anyhow!("Failed to capture standard output from command {}", command) - })?; - let stderr = child.stderr.take().ok_or_else(|| { - anyhow::anyhow!("Failed to capture standard error from command {}", command) - })?; + pub fn args(&mut self, args: I) -> &mut Self + where + I: IntoIterator, + S: AsRef, + { + self.0.args(args); + self + } - let mut stdout_output = String::new(); - let mut stderr_output = String::new(); + pub async fn run_and_capture_output(&mut self) -> Result { + let cmd_display = self.0.as_std().get_program().to_string_lossy().into_owned(); + let args_display = self.0.as_std().get_args().map(|s| s.to_string_lossy()).join(" "); + + info!("Running command: {cmd_display} {args_display}"); + + // Setup signal handling to terminate the child process + let (tx, rx) = tokio::sync::oneshot::channel(); + + let mut sigterm = signal(SignalKind::terminate())?; + let mut sigint = signal(SignalKind::interrupt())?; + let mut sigquit = signal(SignalKind::quit())?; + + tokio::spawn(async move { + tokio::select! { + _ = sigterm.recv() => { + let _ = tx.send(()); + } + _ = sigint.recv() => { + let _ = tx.send(()); + } + _ = sigquit.recv() => { + let _ = tx.send(()); + } + } + }); - let stdout_writer = io::stdout(); - let stderr_writer = io::stderr(); + let mut child = self.0.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?; - let stdout_future = pipe_output(stdout, stdout_writer, &mut stdout_output); - let stderr_future = pipe_error_output(stderr, stderr_writer, &mut stderr_output); + let stdout = child.stdout.take().ok_or_else(|| { + anyhow::anyhow!("Failed to capture standard output from command {cmd_display}") + })?; + let stderr = child.stderr.take().ok_or_else(|| { + anyhow::anyhow!("Failed to capture standard error from command {cmd_display}") + })?; - let combined_future = try_join(stdout_future, stderr_future); + let mut stdout_output = String::new(); + let mut stderr_output = String::new(); - tokio::select! { - output = combined_future => { - output?; - } - _ = rx => { - let _ = child.kill().await; - return Err(anyhow::anyhow!("Command {} was terminated by signal", command)); - } - } + let stdout_writer = io::stdout(); + let stderr_writer = io::stderr(); - let status = child.wait().await?; - if !status.success() { - return Err(anyhow::anyhow!( - "Command {} failed with args {:?}\nError Output: {}", - command, - args, - stderr_output - )); - } + let stdout_future = pipe_output(stdout, stdout_writer, &mut stdout_output); + let stderr_future = pipe_error_output(stderr, stderr_writer, &mut stderr_output); - Ok(stdout_output) -} + let combined_future = try_join(stdout_future, stderr_future); -/// Runs a command, piping its output to stdout and stderr, and returns the stdout output if successful. -pub async fn spawn_command( - command: String, - args: Vec, -) -> Result<(Option, JoinHandle>)> { - // print command out with args joined by space - tracing::info!("spawn command: {} {}", command, args.join(" ")); - - let mut child = Command::new(&command) - .args(&args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - - let process_id = child.id(); - let join_handle = tokio::spawn({ - async move { - let stdout = child.stdout.take().ok_or_else(|| { - anyhow::anyhow!("Failed to capture standard output from command {}", command) - })?; - let stderr = child.stderr.take().ok_or_else(|| { - anyhow::anyhow!("Failed to capture standard error from command {}", command) - })?; - - let mut stdout_output = String::new(); - let mut stderr_output = String::new(); - - let stdout_writer = io::stdout(); - let stderr_writer = io::stderr(); - - let stdout_future = pipe_output(stdout, stdout_writer, &mut stdout_output); - let stderr_future = pipe_error_output(stderr, stderr_writer, &mut stderr_output); - - let _ = try_join(stdout_future, stderr_future).await; - - let status = child.wait().await?; - if !status.success() { - return Err(anyhow::anyhow!( - "Command {} spawn failed with args {:?}\nError Output: {}", - command, - args, - stderr_output - )); + tokio::select! { + output = combined_future => { + output?; } + _ = rx => { + let _ = child.kill().await; + return Err(anyhow::anyhow!("Command {cmd_display} was terminated by signal")); + } + } - Ok(stdout_output) + let status = child.wait().await?; + if !status.success() { + return Err(anyhow::anyhow!( + "Command {cmd_display} failed with args {args_display}\nError Output: {}", + stderr_output + )); } - }); - Ok((process_id, join_handle)) + Ok(stdout_output) + } } #[cfg(test)] diff --git a/util/signing/testing/Cargo.toml b/util/signing/testing/Cargo.toml index d139b4e8e..239c35f85 100644 --- a/util/signing/testing/Cargo.toml +++ b/util/signing/testing/Cargo.toml @@ -19,7 +19,7 @@ async-trait = { workspace = true } maptos-dof-execution = { workspace = true } maptos-execution-util = { workspace = true } movement-signing-aptos = { workspace = true } -movement-signer-loader = { workspace = true } +movement-signer-loader = { workspace = true } movement-signer-local = { workspace = true } movement-signer-aws-kms = { workspace = true } movement-signing-eth = { workspace = true } @@ -28,6 +28,7 @@ aptos-types = { workspace = true } anyhow = { workspace = true } chrono = { workspace = true } ed25519-dalek = { workspace = true, features = ["rand_core"] } +hex = { workspace = true } # Workspace is on rand 0.7 due largely to aptos-core rand = "0.8" sha3 = "0.10.8" diff --git a/util/signing/testing/tests/execute.rs b/util/signing/testing/tests/execute.rs index b62ac9f06..35a51bc77 100644 --- a/util/signing/testing/tests/execute.rs +++ b/util/signing/testing/tests/execute.rs @@ -12,15 +12,14 @@ use aptos_types::transaction::{ }; use anyhow::Context; -use aptos_crypto::ValidCryptoMaterialStringExt; use movement_signer_loader::identifiers::{local::Local, SignerIdentifier}; use tempfile::TempDir; -fn setup(mut maptos_config: Config) -> Result<(Executor, TempDir), anyhow::Error> { +async fn setup(mut maptos_config: Config) -> Result<(Executor, TempDir), anyhow::Error> { let tempdir = tempfile::tempdir()?; // replace the db path with the temporary directory maptos_config.chain.maptos_db_path.replace(tempdir.path().to_path_buf()); - let executor = Executor::try_from_config(maptos_config)?; + let executor = Executor::try_from_config(maptos_config).await?; Ok((executor, tempdir)) } @@ -45,11 +44,11 @@ async fn execute_signed_transaction() -> Result<(), anyhow::Error> { let private_key = Ed25519PrivateKey::generate_for_testing(); let mut config = Config::default(); let signing_key = ed25519_dalek::SigningKey::from_bytes(&private_key.to_bytes()); - config.chain.maptos_private_key_signer_identifier = SignerIdentifier::Local(Local { - private_key_hex_bytes: private_key.to_encoded_string()?.to_string(), - }); + let private_key_hex_bytes = hex::encode(&private_key.to_bytes()); + config.chain.maptos_private_key_signer_identifier = + SignerIdentifier::Local(Local { private_key_hex_bytes }); let signer = TestSigner::new(signing_key); - let (executor, _tempdir) = setup(config)?; + let (executor, _tempdir) = setup(config).await?; let transaction = create_signed_transaction(&signer).await?; let block_id = HashValue::random(); let block_metadata = executor