Merge remote-tracking branch 'origin/main' into rm/libp2p-dht-persist…

…ence

.config/nextest.toml

@@ -1,7 +1,7 @@
 [profile.default]
 # Kill tests after 3 periods of 2m, because they are probably hanging
 slow-timeout = { period = "2m", terminate-after = 3 }
-default-filter = 'not (test(slow_) | package(tests))'
+default-filter = 'not (deps(hotshot-testing) | test(slow_) | package(tests))'
 retries = 2
 
 [[profile.default.overrides]]
@@ -20,6 +20,14 @@ retries  = 10
 filter = 'test(slow_test_restart)'
 threads-required = 'num-cpus'
 
+# HotShot integration tests
+[profile.hotshot]
+retries = 2
+slow-timeout = { period = "1m", terminate-after = 3 }
+default-filter = 'deps(hotshot-testing)'
+final-status-level = "flaky"
+threads-required = "num-test-threads"
+
 [profile.slow]
 slow-timeout = "2m"
 default-filter = 'test(slow_)'
@@ -34,3 +42,14 @@ retries = 2
 slow-timeout = "2m"
 default-filter = 'package(tests)'
 retries = 2
+
+# Serial required
+# TRY 3 FAIL [   2.790s] hotshot-testing::tests_4 tests_4::test_marketplace::test_marketplace_builders_down::testtypes_::memoryimpl_::test_marketplace_builders_down
+# TRY 3 FAIL [   4.446s] hotshot-testing::tests_4 tests_4::test_marketplace::test_marketplace_fallback_builder_down::testtypes_::memoryimpl_::test_marketplace_fallback_builder_down
+# TRY 3 FAIL [   5.119s] hotshot-testing::tests_4 tests_4::test_marketplace::test_marketplace_upgrade::testtypes_::memoryimpl_::test_marketplace_upgrade
+#
+# FLAKY 2/3 [  12.037s] hotshot-testing::tests_2 tests_2::catchup::test_all_restart::testtypes_::pushcdnimpl_::test_all_restart
+# FLAKY 2/3 [  49.943s] hotshot-testing::tests_6 tests_6::test_epochs::test_combined_network_cdn_crash_with_epochs::testtypes_::combinedimpl_::test_combined_network_cdn_crash_with_epochs
+# FLAKY 2/3 [   6.968s] hotshot-testing::tests_6 tests_6::test_epochs::test_epoch_end::testtwostaketablestypes_::pushcdnimpl_::test_epoch_end
+# FLAKY 2/3 [ 125.353s] sequencer api::test::test_fee_upgrade_view_based
+# FLAKY 2/3 [   2.870s] sequencer api::test::test_hotshot_event_streaming

.env

@@ -57,7 +57,7 @@ ESPRESSO_BUILDER_ETH_ACCOUNT_INDEX=8
 ESPRESSO_DEPLOYER_ACCOUNT_INDEX=9
 
 # Contracts
-ESPRESSO_SEQUENCER_LIGHT_CLIENT_PROXY_ADDRESS=0xed1db453c3156ff3155a97ad217b3087d5dc5f6e
+ESPRESSO_SEQUENCER_LIGHT_CLIENT_PROXY_ADDRESS=0x0c8e79f3534b00d9a3d4a856b665bf4ebc22f2ba
 ESPRESSO_SEQUENCER_LIGHTCLIENT_ADDRESS=$ESPRESSO_SEQUENCER_LIGHT_CLIENT_PROXY_ADDRESS
 ESPRESSO_SEQUENCER_PERMISSIONED_PROVER=0x14dc79964da2c08b23698b3d3cc7ca32193d9955
 ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS=0x8ce361602b935680e8dec218b820ff5056beb7af

.env.lightweight

@@ -0,0 +1,8 @@
+ESPRESSO_SEQUENCER_LIGHTWEIGHT=true
+ESPRESSO_SEQUENCER_DATABASE_PRUNE=true
+ESPRESSO_SEQUENCER_PRUNER_BATCH_SIZE=5000
+ESPRESSO_SEQUENCER_PRUNER_TARGET_RETENTION=3d
+ESPRESSO_SEQUENCER_PRUNER_MINIMUM_RETENTION=12h
+ESPRESSO_SEQUENCER_PRUNER_INTERVAL=2h
+ESPRESSO_SEQUENCER_PRUNER_MAX_USAGE=7000
+ESPRESSO_SEQUENCER_PRUNER_PRUNING_THRESHOLD="100 GB"

.envrc

@@ -5,3 +5,4 @@ fi
 use nix
 watch_file flake.nix
 watch_file flake.lock
+watch_file rust-toolchain.toml

.github/workflows/build-without-lockfile.yml

@@ -11,6 +11,10 @@ on:
   schedule:
     - cron: "0 0 * * 1"
   pull_request:
+    # Only run on PRs if dependencies were modified.
+    paths:
+      - "**/Cargo.toml"
+      - "**/Cargo.lock"
   workflow_dispatch:
 
 concurrency:
@@ -32,4 +36,53 @@ jobs:
       - name: Build without committed Cargo.lock
         run: |
           cargo generate-lockfile
-          cargo check --all-features --all-targets
+          cargo check --all-targets
+
+      - name: Comment on PR
+        uses: actions/github-script@v7
+        if: failure() && github.event_name == 'pull_request'
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const issue_number = context.issue.number;
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const runUrl = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;
+
+            const commentTitle = "Unable to build without Cargo.lock file"
+            const commentBody = `${commentTitle}.
+
+            This means that after this change 3rd party projects may have
+            difficulties using crates in this repo as a dependency. If this
+            isn't easy to fix please open an issue so we can fix it later.
+
+            For the first failing build see: ${runUrl}
+
+            To reproduce locally run
+
+            \`\`\`
+            cargo generate-lockfile
+            cargo check --all-targets
+            \`\`\`
+
+            This PR can still be merged.`;
+
+            // Fetch existing comments
+            const { data: comments } = await github.rest.issues.listComments({
+              owner,
+              repo,
+              issue_number,
+            });
+
+            // Find existing comment
+            const existingComment = comments.find(c => c.body.startsWith(commentTitle));
+            if (!existingComment) {
+              await github.rest.issues.createComment({
+                owner,
+                repo,
+                issue_number,
+                body: commentBody
+              });
+            } else {
+              console.log("Already commented.")
+            }

.github/workflows/build.yml

@@ -181,6 +181,12 @@ jobs:
           name: aarch64-unknown-linux-gnu-services
           path: target/arm64/release
 
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          # Temporary fix (See https://github.com/docker/setup-qemu-action/issues/198)
+          image: tonistiigi/binfmt:qemu-v7.0.0-28
+
       - name: Setup Docker BuildKit (buildx)
         uses: docker/setup-buildx-action@v3
 

.github/workflows/cargo-features.yml

@@ -13,13 +13,13 @@ on:
   pull_request:
   workflow_dispatch:
 
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
   cargo-features:
-    runs-on: ubuntu-latest
+    runs-on: buildjet-8vcpu-ubuntu-2204
     steps:
       - uses: taiki-e/install-action@cargo-hack
 
@@ -35,8 +35,8 @@ jobs:
       # as each individual feature enabled.
       - name: Check compilation for feature combinations
         run: |
-          cargo hack check --feature-powerset
+          cargo hack check --feature-powerset --exclude hotshot --exclude hotshot-builder-api --exclude hotshot-task-impls --exclude hotshot-macros --exclude hotshot-events-service --exclude hotshot-utils --exclude hotshot-orchestrator --exclude hotshot-query-service --exclude hotshot-stake-table --exclude hotshot-state-prover --exclude hotshot-task --exclude hotshot-testing --exclude hotshot-types --exclude hotshot-libp2p-networking --exclude hotshot-contract-adapter --exclude hotshot-example-types --exclude vid
 
       - name: Check compilation for feature combinations (--tests)
         run: |
-          cargo hack check --feature-powerset --tests
+          cargo hack check --feature-powerset --tests --exclude hotshot --exclude hotshot-builder-api --exclude hotshot-task-impls --exclude hotshot-macros --exclude hotshot-events-service --exclude hotshot-utils --exclude hotshot-orchestrator --exclude hotshot-query-service --exclude hotshot-stake-table --exclude hotshot-state-prover --exclude hotshot-task --exclude hotshot-testing --exclude hotshot-types --exclude hotshot-libp2p-networking --exclude hotshot-contract-adapter --exclude hotshot-example-types --exclude vid

.github/workflows/contracts.yml

@@ -21,7 +21,7 @@ env:
 
 jobs:
   contracts:
-    timeout-minutes: 30
+    timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:
       - name: Install Nix

.github/workflows/doc-rust.yml

@@ -20,8 +20,7 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v4
 
-      - name: Install Just
-        run: sudo apt-get install -y just
+      - uses: taiki-e/install-action@just
 
       - uses: Swatinem/rust-cache@v2
         name: Enable Rust Caching

.github/workflows/hotshot.yml

@@ -0,0 +1,93 @@
+name: HotShot integration tests
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+  schedule:
+    - cron: "0 0 * * 1"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        test_suites:
+          # - test-ci-1 temporarily disabled for (https://github.com/EspressoSystems/espresso-sequencer/issues/2664)
+          - test-ci-2
+          - test-ci-3
+          - test-ci-4
+          - test-ci-5
+          - test-ci-6-1
+          - test-ci-6-2
+          - test-ci-6-3
+          - test-ci-6-4
+          - test-ci-6-5
+          - test-ci-6-6
+          - test-ci-rest
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout Repository
+
+      - name: Install Rust
+        uses: mkroening/rust-toolchain-toml@main
+
+      - uses: Swatinem/rust-cache@v2
+        name: Enable Rust Caching
+        with:
+          shared-key: "hotshot-tests"
+          cache-on-failure: "true"
+          save-if: ${{ github.ref == 'refs/heads/main' && matrix.test_suites == 'test-ci-rest' }}
+          prefix-key: v0-hotshot
+
+      - uses: taiki-e/install-action@just
+
+      - uses: taiki-e/install-action@nextest
+
+      - name: Show memory
+        run: free -h
+
+      - name: Collect Workflow Telemetry
+        uses: catchpoint/workflow-telemetry-action@v2
+        with:
+          comment_on_pr: false
+
+      - name: Unit and integration tests for all crates in workspace
+        run: |
+          just hotshot::${{ matrix.test_suites }}
+        timeout-minutes: 60
+        env:
+          RUST_BACKTRACE: full
+
+  test-examples:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout Repository
+
+      - name: Install Rust
+        uses: mkroening/rust-toolchain-toml@main
+
+      - uses: Swatinem/rust-cache@v2
+        name: Enable Rust Caching
+        with:
+          shared-key: "hotshot-examples"
+          cache-on-failure: "true"
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+          prefix-key: v0-hotshot
+
+      - uses: taiki-e/install-action@just
+
+      - name: Test examples
+        run: |
+          just hotshot::example all-push-cdn -- --config_file ./hotshot-orchestrator/run-config.toml
+        timeout-minutes: 20

.github/workflows/nix-env-macos-arm.yml

@@ -0,0 +1,58 @@
+# This workflow ensures that the nix dev env works on ARM based macOS.
+name: Nix env macOS ARM
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+    tags:
+      # YYYYMMDD
+      - "20[0-9][0-9][0-1][0-9][0-3][0-9]*"
+  pull_request:
+    branches:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUST_LOG: info,libp2p=off
+
+jobs:
+  nix-env-macos-arm:
+    timeout-minutes: 60
+    runs-on: macos-latest
+    steps:
+      - name: Install Nix
+        uses: cachix/install-nix-action@v30
+
+      - name: Enable Cachix
+        uses: cachix/cachix-action@v15
+        # If PR is from a non-collaborator (e. g. dependabot) the secrets are missing and the login to cachix fails.
+        continue-on-error: true
+        with:
+          name: espresso-systems-private
+          authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}"
+          extraPullNames: nix-community
+          skipPush: ${{ github.actor == 'dependabot[bot]' }}
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: Swatinem/rust-cache@v2
+        name: Enable Rust Caching
+        with:
+          prefix-key: v0-rust
+
+      - name: Activate nix env
+        run: nix develop --accept-flake-config -c echo Ok
+
+      - name: Build rust crates
+        run: nix develop --accept-flake-config -c just build
+
+      - name: Build contracts
+        run: nix develop --accept-flake-config -c forge build