From 52749511911482c06cefd87e0e6ff211ae356917 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 21 Jan 2025 18:39:02 +0500 Subject: [PATCH 01/77] epoch integration test --- .env | 1 + Cargo.lock | 2 + client/Cargo.toml | 1 + client/src/lib.rs | 19 ++++- data/genesis/demo-epoch.toml | 2 +- justfile | 4 + process-compose.yaml | 4 +- sequencer-sqlite/Cargo.lock | 1 + sequencer/api/node.toml | 4 + sequencer/src/api.rs | 8 ++ sequencer/src/api/data_source.rs | 2 + sequencer/src/run.rs | 21 ++---- tests/Cargo.toml | 1 + tests/upgrades.rs | 125 ++++++++++++++++++++++++++++--- types/src/v0/mod.rs | 2 +- utils/src/stake_table.rs | 18 ++++- 16 files changed, 181 insertions(+), 34 deletions(-) diff --git a/.env b/.env index d716154cfb..84d24e9100 100644 --- a/.env +++ b/.env @@ -152,3 +152,4 @@ INTEGRATION_TEST_SEQUENCER_VERSION=02 # max database connections ESPRESSO_SEQUENCER_DATABASE_MAX_CONNECTIONS=25 + \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index a1f5e28679..eddeaacf4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1731,6 +1731,7 @@ dependencies = [ "espresso-types", "ethers", "futures", + "hotshot-types", "jf-merkle-tree", "surf-disco", "tokio", @@ -10028,6 +10029,7 @@ dependencies = [ "ethers", "futures", "reqwest 0.12.12", + "sequencer-utils", "surf-disco", "tokio", "vbs", diff --git a/client/Cargo.toml b/client/Cargo.toml index 85db2033cc..c38a613844 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -15,3 +15,4 @@ surf-disco = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } vbs = { workspace = true } +hotshot-types = { workspace = true } \ No newline at end of file diff --git a/client/src/lib.rs b/client/src/lib.rs index a9d5cc995f..572d2e796e 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1,7 +1,8 @@ use anyhow::Context; -use espresso_types::{FeeAccount, FeeAmount, FeeMerkleTree, Header}; +use espresso_types::{FeeAccount, FeeAmount, FeeMerkleTree, Header, PubKey}; use ethers::types::Address; use futures::{stream::BoxStream, StreamExt}; +use hotshot_types::stake_table::StakeTableEntry; use jf_merkle_tree::{ prelude::{MerkleProof, Sha3Node}, MerkleTreeScheme, @@ -119,6 +120,22 @@ impl SequencerClient { let balance = proof.elem().copied().unwrap_or(0.into()); Ok(balance) } + + pub async fn current_epoch(&self) -> anyhow::Result { + self.0 + .get::("node/current_epoch") + .send() + .await + .context("getting epoch value") + } + + pub async fn stake_table(&self, epoch: u64) -> anyhow::Result>> { + self.0 + .get::<_>(&format!("node/stake-table/{epoch}")) + .send() + .await + .context("getting epoch value") + } } #[cfg(test)] diff --git a/data/genesis/demo-epoch.toml b/data/genesis/demo-epoch.toml index 5b351243f5..09f60756ab 100644 --- a/data/genesis/demo-epoch.toml +++ b/data/genesis/demo-epoch.toml @@ -18,7 +18,7 @@ timestamp = "1970-01-01T00:00:00Z" number = 0 [[upgrade]] -version = "0.99" +version = "0.3" start_proposing_view = 10 stop_proposing_view = 60 diff --git a/justfile b/justfile index 0424edadd5..4968621575 100644 --- a/justfile +++ b/justfile @@ -29,6 +29,10 @@ build profile="test": demo-native-mp *args: build scripts/demo-native -f process-compose.yaml -f process-compose-mp.yml {{args}} +demo-native-epoch *args: build + # export DEMO_GENESIS_FILE=data/genesis/demo-epoch.toml + scripts/demo-native -f process-compose.yaml {{args}} + demo-native-benchmark: cargo build --release --features benchmarking scripts/demo-native diff --git a/process-compose.yaml b/process-compose.yaml index 281319cfff..d2fa812225 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -5,8 +5,8 @@ environment: - ESPRESSO_SEQUENCER_ORCHESTRATOR_URL=http://localhost:$ESPRESSO_ORCHESTRATOR_PORT - ESPRESSO_SEQUENCER_URL=http://localhost:$ESPRESSO_SEQUENCER_API_PORT - ESPRESSO_SEQUENCER_L1_PROVIDER=http://localhost:$ESPRESSO_SEQUENCER_L1_PORT - - ESPRESSO_SEQUENCER_GENESIS_FILE=data/genesis/demo.toml - - ESPRESSO_BUILDER_GENESIS_FILE=data/genesis/demo.toml + - ESPRESSO_SEQUENCER_GENESIS_FILE=data/genesis/demo-epoch.toml + - ESPRESSO_BUILDER_GENESIS_FILE=data/genesis/demo-epoch.toml - ESPRESSO_SEQUENCER_INITIAL_PERMISSIONED_STAKE_TABLE_PATH=data/initial_stake_table.toml - ESPRESSO_STATE_RELAY_SERVER_URL=http://localhost:$ESPRESSO_STATE_RELAY_SERVER_PORT - QUERY_SERVICE_URI=http://localhost:$ESPRESSO_SEQUENCER1_API_PORT/v0/ diff --git a/sequencer-sqlite/Cargo.lock b/sequencer-sqlite/Cargo.lock index d7b35400fb..920a60ce78 100644 --- a/sequencer-sqlite/Cargo.lock +++ b/sequencer-sqlite/Cargo.lock @@ -1661,6 +1661,7 @@ dependencies = [ "espresso-types", "ethers", "futures", + "hotshot-types", "jf-merkle-tree", "surf-disco", "tokio", diff --git a/sequencer/api/node.toml b/sequencer/api/node.toml index ddcdaa7db6..d2025500f7 100644 --- a/sequencer/api/node.toml +++ b/sequencer/api/node.toml @@ -6,3 +6,7 @@ DOC = "Get the stake table for the current epoch" PATH = ["stake-table/:epoch_number"] ":epoch_number" = "Integer" DOC = "Get the stake table for the given epoch" + +[route.current_epoch] +PATH = ["current_epoch"] +DOC = "Get the current epoch" \ No newline at end of file diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index dee017a6fa..7e4b227556 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -170,6 +170,10 @@ impl, D: Sync, V: Versions, P: SequencerPersistence> ) -> Vec::SignatureKey>> { self.as_ref().get_stake_table(epoch).await } + + async fn get_current_epoch(&self) -> ::Epoch { + self.as_ref().get_current_epoch().await + } } impl, V: Versions, P: SequencerPersistence> @@ -196,6 +200,10 @@ impl, V: Versions, P: SequencerPersistence> .await .stake_table(epoch) } + + async fn get_current_epoch(&self) -> ::Epoch { + self.consensus().await.read().await.cur_epoch().await + } } impl, V: Versions, P: SequencerPersistence> SubmitDataSource diff --git a/sequencer/src/api/data_source.rs b/sequencer/src/api/data_source.rs index d26dce301c..2029871687 100644 --- a/sequencer/src/api/data_source.rs +++ b/sequencer/src/api/data_source.rs @@ -122,6 +122,8 @@ pub(crate) trait StakeTableDataSource { &self, epoch: Option<::Epoch>, ) -> impl Send + Future>>; + + fn get_current_epoch(&self) -> impl Send + Future::Epoch>; } pub(crate) trait CatchupDataSource: Sync { diff --git a/sequencer/src/run.rs b/sequencer/src/run.rs index 59cf5dcafc..d1cf4e3425 100644 --- a/sequencer/src/run.rs +++ b/sequencer/src/run.rs @@ -9,8 +9,8 @@ use super::{ }; use clap::Parser; use espresso_types::{ - traits::NullEventConsumer, FeeVersion, MarketplaceVersion, SequencerVersions, - SolverAuctionResultsProvider, V0_0, + traits::NullEventConsumer, EpochVersion, FeeVersion, MarketplaceVersion, SequencerVersions, + SolverAuctionResultsProvider, }; use futures::future::FutureExt; use hotshot::MarketplaceConfig; @@ -38,30 +38,21 @@ pub async fn main() -> anyhow::Result<()> { let upgrade = genesis.upgrade_version; match (base, upgrade) { - (FeeVersion::VERSION, MarketplaceVersion::VERSION) => { + (FeeVersion::VERSION, EpochVersion::VERSION) => { run( genesis, modules, opt, - SequencerVersions::::new(), + SequencerVersions::::new(), ) .await } - (FeeVersion::VERSION, _) => { - run( - genesis, - modules, - opt, - SequencerVersions::::new(), - ) - .await - } - (MarketplaceVersion::VERSION, _) => { + (FeeVersion::VERSION, MarketplaceVersion::VERSION) => { run( genesis, modules, opt, - SequencerVersions::::new(), + SequencerVersions::::new(), ) .await } diff --git a/tests/Cargo.toml b/tests/Cargo.toml index eeb91c14a7..09ae39b76c 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -20,3 +20,4 @@ reqwest = { workspace = true, features = ["json"] } surf-disco = { workspace = true } tokio = { workspace = true } vbs = { workspace = true } +sequencer-utils = { path = "../utils" } \ No newline at end of file diff --git a/tests/upgrades.rs b/tests/upgrades.rs index 5d68938440..93e61bf347 100644 --- a/tests/upgrades.rs +++ b/tests/upgrades.rs @@ -1,8 +1,14 @@ +use std::{path::Path, time::Duration}; + use crate::common::TestConfig; use anyhow::Result; -use espresso_types::{FeeVersion, MarketplaceVersion}; +use client::SequencerClient; +use dotenvy::var; +use espresso_types::{EpochVersion, FeeVersion, MarketplaceVersion}; use futures::{future::join_all, StreamExt}; -use vbs::version::StaticVersionType; +use sequencer_utils::stake_table::{update_stake_table, PermissionedStakeTableUpdate}; +use tokio::time::sleep; +use vbs::version::{StaticVersionType, Version}; const SEQUENCER_BLOCKS_TIMEOUT: u64 = 200; @@ -12,10 +18,10 @@ async fn test_upgrade() -> Result<()> { let testing = TestConfig::new().await.unwrap(); - let versions = if testing.sequencer_version >= 3 { - (FeeVersion::version(), MarketplaceVersion::version()) - } else { - panic!("Invalid sequencer version provided for upgrade test."); + let (base, upgrade) = match testing.sequencer_version { + 3 => (FeeVersion::version(), EpochVersion::version()), + version if version > 3 => (FeeVersion::version(), MarketplaceVersion::version()), + _ => panic!("Invalid sequencer version provided for upgrade test."), }; println!("Waiting on readiness"); @@ -26,6 +32,23 @@ async fn test_upgrade() -> Result<()> { let clients = testing.sequencer_clients; + let height = test_header_version(clients.clone(), base, upgrade).await?; + // check that atleast 50 blocks are produced after the upgrade + test_blocks_production(clients.clone(), height, 50).await?; + + if upgrade == EpochVersion::version() { + test_stake_table_update(clients).await?; + } + + // TODO assert transactions are incrementing + Ok(()) +} + +async fn test_header_version( + clients: Vec, + base: Version, + upgrade: Version, +) -> Result { // Test is limited to those sequencers with correct modules // enabled. It would be less fragile if we could discover them. let subscriptions = join_all(clients.iter().map(|c| c.subscribe_headers(0))) @@ -34,7 +57,7 @@ async fn test_upgrade() -> Result<()> { .collect::>>()?; let mut stream = futures::stream::iter(subscriptions).flatten_unordered(None); - + let mut height = 0; while let Some(header) = stream.next().await { let header = header.unwrap(); println!( @@ -46,11 +69,12 @@ async fn test_upgrade() -> Result<()> { // TODO is it possible to discover the view at which upgrade should be finished? // First few views should be `Base` version. if header.height() <= 20 { - assert_eq!(header.version(), versions.0) + assert_eq!(header.version(), base) } - if header.version() == versions.1 { + if header.version() == upgrade { println!("header version matched! height={:?}", header.height()); + height = header.height(); break; } @@ -59,6 +83,87 @@ async fn test_upgrade() -> Result<()> { } } - // TODO assert transactions are incrementing + Ok(height) +} + +async fn test_blocks_production(clients: Vec, from: u64, num: u64) -> Result<()> { + let subscriptions = join_all(clients.iter().map(|c| c.subscribe_blocks(from))) + .await + .into_iter() + .collect::>>()?; + + let mut num_blocks = 0; + + for mut node in subscriptions { + while let Some(block) = node.next().await { + let _block = block.unwrap(); + num_blocks += 1; + if num_blocks == num { + break; + } + } + + num_blocks = 0; + } + + Ok(()) +} + +async fn test_stake_table_update(clients: Vec) -> Result<()> { + /* + EPOCH V3 + */ + + let rpc_url = var("ESPRESSO_SEQUENCER_L1_PROVIDER")?; + let account_index = var("ESPRESSO_DEPLOYER_ACCOUNT_INDEX")?; + let contract_address = var("ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS")?; + let initial_stake_table_path = var("ESPRESSO_SEQUENCER_INITIAL_PERMISSIONED_STAKE_TABLE_PATH")?; + + let permissioned_stake_table = + PermissionedStakeTableUpdate::from_toml_file(Path::new(&initial_stake_table_path))?; + + // initial stake table has 5 new stakers + + let new_stakers = permissioned_stake_table.new_stakers; + //lets remove one + let staker_removed = new_stakers[0].clone(); + + let st_with_one_removed = + PermissionedStakeTableUpdate::new(vec![staker_removed.clone()], vec![]); + let client = clients[0].clone(); + + let epoch_before_update = client.current_epoch().await?; + + update_stake_table( + rpc_url.parse()?, + Duration::from_secs(7), + "test test test test test test test test test test test junk".to_string(), + account_index.parse()?, + contract_address.parse()?, + st_with_one_removed, + ) + .await?; + + loop { + sleep(Duration::from_secs(10)).await; + let epoch = clients[0].current_epoch().await?; + + if epoch > epoch_before_update { + let stake_table = client.stake_table(epoch).await?; + assert_eq!(stake_table.len(), 4); + + assert!( + stake_table + .iter() + .all(|st| st.stake_key != staker_removed.stake_table_key), + "Entry for {} already exists in the stake table", + staker_removed.stake_table_key + ); + + break; + } + } + // TODO: randomize this test + Ok(()) } diff --git a/types/src/v0/mod.rs b/types/src/v0/mod.rs index 40c3811a6a..8275709128 100644 --- a/types/src/v0/mod.rs +++ b/types/src/v0/mod.rs @@ -178,8 +178,8 @@ pub type MockSequencerVersions = SequencerVersions, StaticVe pub type V0_0 = StaticVersion<0, 0>; pub type V0_1 = StaticVersion<0, 1>; pub type FeeVersion = StaticVersion<0, 2>; +pub type EpochVersion = StaticVersion<0, 3>; pub type MarketplaceVersion = StaticVersion<0, 99>; -pub type EpochVersion = StaticVersion<0, 100>; pub type Leaf = hotshot_types::data::Leaf; pub type Leaf2 = hotshot_types::data::Leaf2; diff --git a/utils/src/stake_table.rs b/utils/src/stake_table.rs index 94a3eac1d9..b303d070e5 100644 --- a/utils/src/stake_table.rs +++ b/utils/src/stake_table.rs @@ -57,8 +57,8 @@ impl From for Vec { } #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, From, PartialEq)] -struct StakerIdentity { - stake_table_key: BLSPubKey, +pub struct StakerIdentity { + pub stake_table_key: BLSPubKey, } impl From for BLSPubKey { @@ -72,12 +72,22 @@ impl From for BLSPubKey { #[serde(bound(deserialize = ""))] pub struct PermissionedStakeTableUpdate { #[serde(default)] - stakers_to_remove: Vec, + pub stakers_to_remove: Vec, #[serde(default)] - new_stakers: Vec>, + pub new_stakers: Vec>, } impl PermissionedStakeTableUpdate { + pub fn new( + new_stakers: Vec>, + stakers_to_remove: Vec, + ) -> Self { + Self { + stakers_to_remove, + new_stakers, + } + } + pub fn from_toml_file(path: &Path) -> anyhow::Result { let config_file_as_string: String = fs::read_to_string(path) .unwrap_or_else(|_| panic!("Could not read config file located at {}", path.display())); From 5e27fc281b23c65487c8ceeee5ff7b19f325aaa0 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Thu, 30 Jan 2025 18:22:01 +0500 Subject: [PATCH 02/77] DEMO_GENESIS_FILE env --- justfile | 3 +-- process-compose.yaml | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/justfile b/justfile index 05c8e2881a..b8eb2ad79c 100644 --- a/justfile +++ b/justfile @@ -30,8 +30,7 @@ demo-native-mp *args: build scripts/demo-native -f process-compose.yaml -f process-compose-mp.yml {{args}} demo-native-epoch *args: build - # export DEMO_GENESIS_FILE=data/genesis/demo-epoch.toml - scripts/demo-native -f process-compose.yaml {{args}} + DEMO_GENESIS_FILE=data/genesis/demo-epoch.toml scripts/demo-native -f process-compose.yaml {{args}} demo-native-benchmark: cargo build --release --features benchmarking diff --git a/process-compose.yaml b/process-compose.yaml index d2fa812225..d50605c2ee 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -5,8 +5,8 @@ environment: - ESPRESSO_SEQUENCER_ORCHESTRATOR_URL=http://localhost:$ESPRESSO_ORCHESTRATOR_PORT - ESPRESSO_SEQUENCER_URL=http://localhost:$ESPRESSO_SEQUENCER_API_PORT - ESPRESSO_SEQUENCER_L1_PROVIDER=http://localhost:$ESPRESSO_SEQUENCER_L1_PORT - - ESPRESSO_SEQUENCER_GENESIS_FILE=data/genesis/demo-epoch.toml - - ESPRESSO_BUILDER_GENESIS_FILE=data/genesis/demo-epoch.toml + - ESPRESSO_SEQUENCER_GENESIS_FILE=$DEMO_GENESIS_FILE + - ESPRESSO_BUILDER_GENESIS_FILE=$DEMO_GENESIS_FILE - ESPRESSO_SEQUENCER_INITIAL_PERMISSIONED_STAKE_TABLE_PATH=data/initial_stake_table.toml - ESPRESSO_STATE_RELAY_SERVER_URL=http://localhost:$ESPRESSO_STATE_RELAY_SERVER_PORT - QUERY_SERVICE_URI=http://localhost:$ESPRESSO_SEQUENCER1_API_PORT/v0/ From a1803f7793baa9160fd3382bd9c773165354ba5e Mon Sep 17 00:00:00 2001 From: tbro Date: Mon, 3 Feb 2025 18:42:52 -0300 Subject: [PATCH 03/77] fix error en `.env` file --- .env | 1 - 1 file changed, 1 deletion(-) diff --git a/.env b/.env index 84d24e9100..d716154cfb 100644 --- a/.env +++ b/.env @@ -152,4 +152,3 @@ INTEGRATION_TEST_SEQUENCER_VERSION=02 # max database connections ESPRESSO_SEQUENCER_DATABASE_MAX_CONNECTIONS=25 - \ No newline at end of file From d688b6c665fd41c3efef0314c178132a7f17469a Mon Sep 17 00:00:00 2001 From: tbro Date: Mon, 3 Feb 2025 19:09:47 -0300 Subject: [PATCH 04/77] add pos test to justfile and CI workflow --- .github/workflows/test.yml | 7 +++++-- justfile | 7 ++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 852dd0b9fe..7ce93d29b2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -182,11 +182,14 @@ jobs: needs: [build-test-bins, build-test-artifacts-postgres] strategy: matrix: - version: [02,99] + version: [02,03,99] include: - version: 02 compose: "-f process-compose.yaml -D" - + - version: 03 + env: + DEMO_GENESIS_FILE: data/genesis/demo-epoch.toml + compose: "-f process-compose.yaml -D" - version: 99 compose: "-f process-compose.yaml -f process-compose-mp.yml -D" runs-on: ubuntu-latest diff --git a/justfile b/justfile index b8eb2ad79c..2005f5693d 100644 --- a/justfile +++ b/justfile @@ -29,7 +29,7 @@ build profile="test": demo-native-mp *args: build scripts/demo-native -f process-compose.yaml -f process-compose-mp.yml {{args}} -demo-native-epoch *args: build +demo-native-pos *args: build DEMO_GENESIS_FILE=data/genesis/demo-epoch.toml scripts/demo-native -f process-compose.yaml {{args}} demo-native-benchmark: @@ -74,10 +74,15 @@ test-all: test-integration: @echo 'NOTE that demo-native must be running for this test to succeed.' INTEGRATION_TEST_SEQUENCER_VERSION=2 cargo nextest run --all-features --nocapture --profile integration smoke + test-integration-mp: @echo 'NOTE that demo-native-mp must be running for this test to succeed.' INTEGRATION_TEST_SEQUENCER_VERSION=99 cargo nextest run --all-features --nocapture --profile integration +test-integration-pos: + @echo 'NOTE that demo-native-pos must be running for this test to succeed.' + INTEGRATION_TEST_SEQUENCER_VERSION=3 cargo nextest run --all-features --nocapture --profile integration smoke + clippy: @echo 'features: "embedded-db"' cargo clippy --workspace --features embedded-db --all-targets -- -D warnings From 697d02246e2fda5401ad132978a7800e76cdb53f Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 4 Feb 2025 19:12:26 +0500 Subject: [PATCH 05/77] run demo native with base version set to pos --- .github/workflows/test.yml | 2 +- data/genesis/demo-pos-base.toml | 21 +++++++++++++++++++ .../{demo-epoch.toml => demo-pos.toml} | 0 justfile | 5 ++++- sequencer/src/run.rs | 11 +++++++++- 5 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 data/genesis/demo-pos-base.toml rename data/genesis/{demo-epoch.toml => demo-pos.toml} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7ce93d29b2..c4dda28b0f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -188,7 +188,7 @@ jobs: compose: "-f process-compose.yaml -D" - version: 03 env: - DEMO_GENESIS_FILE: data/genesis/demo-epoch.toml + DEMO_GENESIS_FILE: data/genesis/demo-pos.toml compose: "-f process-compose.yaml -D" - version: 99 compose: "-f process-compose.yaml -f process-compose-mp.yml -D" diff --git a/data/genesis/demo-pos-base.toml b/data/genesis/demo-pos-base.toml new file mode 100644 index 0000000000..38fb3deff5 --- /dev/null +++ b/data/genesis/demo-pos-base.toml @@ -0,0 +1,21 @@ +base_version = "0.3" +upgrade_version = "0.0" + +[stake_table] +capacity = 10 + +[chain_config] +chain_id = 999999999 +max_block_size = '1mb' +base_fee = '1 wei' +fee_recipient = "0x0000000000000000000000000000000000000000" +bid_recipient = "0x0000000000000000000000000000000000000000" +fee_contract = "0xa15bb66138824a1c7167f5e85b957d04dd34e468" +stake_table_contract = "0x8ce361602b935680e8dec218b820ff5056beb7af" + +[header] +timestamp = "1970-01-01T00:00:00Z" + +[l1_finalized] +number = 0 + \ No newline at end of file diff --git a/data/genesis/demo-epoch.toml b/data/genesis/demo-pos.toml similarity index 100% rename from data/genesis/demo-epoch.toml rename to data/genesis/demo-pos.toml diff --git a/justfile b/justfile index 2005f5693d..1877c7eca3 100644 --- a/justfile +++ b/justfile @@ -30,7 +30,10 @@ demo-native-mp *args: build scripts/demo-native -f process-compose.yaml -f process-compose-mp.yml {{args}} demo-native-pos *args: build - DEMO_GENESIS_FILE=data/genesis/demo-epoch.toml scripts/demo-native -f process-compose.yaml {{args}} + DEMO_GENESIS_FILE=data/genesis/demo-pos.toml scripts/demo-native -f process-compose.yaml {{args}} + +demo-native-pos-base *args: build + DEMO_GENESIS_FILE=data/genesis/demo-pos-base.toml scripts/demo-native -f process-compose.yaml {{args}} demo-native-benchmark: cargo build --release --features benchmarking diff --git a/sequencer/src/run.rs b/sequencer/src/run.rs index d1cf4e3425..7eee2e2896 100644 --- a/sequencer/src/run.rs +++ b/sequencer/src/run.rs @@ -10,7 +10,7 @@ use super::{ use clap::Parser; use espresso_types::{ traits::NullEventConsumer, EpochVersion, FeeVersion, MarketplaceVersion, SequencerVersions, - SolverAuctionResultsProvider, + SolverAuctionResultsProvider, V0_0, }; use futures::future::FutureExt; use hotshot::MarketplaceConfig; @@ -47,6 +47,15 @@ pub async fn main() -> anyhow::Result<()> { ) .await } + (EpochVersion::VERSION, _) => { + run( + genesis, + modules, + opt, + SequencerVersions::::new(), + ) + .await + } (FeeVersion::VERSION, MarketplaceVersion::VERSION) => { run( genesis, From 7786cfb2394604270fc78f4ebf26a5659e74bb68 Mon Sep 17 00:00:00 2001 From: tbro Date: Tue, 4 Feb 2025 11:48:52 -0300 Subject: [PATCH 06/77] Marketplace signatures should engage @ v99 --- types/src/v0/impls/state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/src/v0/impls/state.rs b/types/src/v0/impls/state.rs index 88d866c54c..a9a3ebc7e6 100644 --- a/types/src/v0/impls/state.rs +++ b/types/src/v0/impls/state.rs @@ -667,7 +667,7 @@ fn validate_builder_fee( // TODO Marketplace signatures are placeholders for now. In // finished Marketplace signatures will cover the full // transaction. - if version.minor >= 3 { + if version.minor >= 99 { fee_info .account() .validate_sequencing_fee_signature_marketplace( From 5f1a2b8074143a8b0a4289cd8bfa31749af68ef4 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 4 Feb 2025 21:57:36 +0500 Subject: [PATCH 07/77] set epoch-height to 150 --- builder/src/lib.rs | 2 +- hotshot-query-service/examples/simple-server.rs | 2 +- hotshot-query-service/src/testing/consensus.rs | 2 +- sequencer/src/lib.rs | 5 ++++- types/src/v0/traits.rs | 3 ++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/builder/src/lib.rs b/builder/src/lib.rs index a8805a7da6..0cb5a108f0 100755 --- a/builder/src/lib.rs +++ b/builder/src/lib.rs @@ -138,7 +138,7 @@ pub mod testing { start_voting_time: 0, stop_proposing_time: 0, stop_voting_time: 0, - epoch_height: 0, + epoch_height: 150, }; Self { diff --git a/hotshot-query-service/examples/simple-server.rs b/hotshot-query-service/examples/simple-server.rs index d22c341858..aeb907d9c7 100644 --- a/hotshot-query-service/examples/simple-server.rs +++ b/hotshot-query-service/examples/simple-server.rs @@ -216,7 +216,7 @@ async fn init_consensus( stop_proposing_time: 0, start_voting_time: 0, stop_voting_time: 0, - epoch_height: 0, + epoch_height: 150, }; let nodes = join_all(priv_keys.into_iter().zip(data_sources).enumerate().map( diff --git a/hotshot-query-service/src/testing/consensus.rs b/hotshot-query-service/src/testing/consensus.rs index e8b94af9ef..06d7565fa3 100644 --- a/hotshot-query-service/src/testing/consensus.rs +++ b/hotshot-query-service/src/testing/consensus.rs @@ -143,7 +143,7 @@ impl MockNetwork { stop_proposing_time: 0, start_voting_time: 0, stop_voting_time: 0, - epoch_height: 0, + epoch_height: 150, }; update_config(&mut config); diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 4844fda77c..ef34353217 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -361,6 +361,9 @@ pub async fn init_node( upgrade.set_hotshot_config_parameters(&mut network_config.config); } + //todo(abdul): get from genesis file + network_config.config.epoch_height = 150; + // If the `Libp2p` bootstrap nodes were supplied via the command line, override those // present in the config file. if let Some(bootstrap_nodes) = network_params.libp2p_bootstrap_nodes { @@ -806,7 +809,7 @@ pub mod testing { start_voting_time: 0, stop_proposing_time: 0, stop_voting_time: 0, - epoch_height: 0, + epoch_height: 150, }; Self { diff --git a/types/src/v0/traits.rs b/types/src/v0/traits.rs index 7f352fdbd5..66e287e01f 100644 --- a/types/src/v0/traits.rs +++ b/types/src/v0/traits.rs @@ -582,7 +582,8 @@ pub trait SequencerPersistence: Sized + Send + Sync + Clone + 'static { Ok(( HotShotInitializer { instance_state: state, - epoch_height: 0, + // todo(abdul): load from storage? + epoch_height: 150, anchor_leaf: leaf, anchor_state: validated_state.unwrap_or_default(), anchor_state_delta: None, From cac2b3b07a81dae816c17b3557b69914a2cff41a Mon Sep 17 00:00:00 2001 From: tbro Date: Tue, 4 Feb 2025 16:45:31 -0300 Subject: [PATCH 08/77] process-compose: remove fund-builder condition from `permissionless-builder` --- .github/workflows/test.yml | 2 +- process-compose.yaml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c4dda28b0f..645d051217 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -226,7 +226,7 @@ jobs: NEXTEST_PROFILE: integration INTEGRATION_TEST_SEQUENCER_VERSION: ${{ matrix.version }} run: | - cargo nextest run --archive-file nextest-archive-postgres.tar.zst --verbose --no-fail-fast --nocapture \ + cargo nextest run --archive-file nextest-archive-postgres.tar.zst --verbose --no-fail-fast \ --workspace-remap $PWD $(if [ "${{ matrix.version }}" == "2" ]; then echo " smoke"; fi) timeout-minutes: 10 diff --git a/process-compose.yaml b/process-compose.yaml index d50605c2ee..558f3d918e 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -522,15 +522,13 @@ processes: depends_on: sequencer0: condition: process_healthy - fund-builder: - condition: process_completed readiness_probe: http_get: scheme: http host: localhost port: $ESPRESSO_BUILDER_SERVER_PORT path: /healthcheck - failure_threshold: 100 + failure_threshold: 5 period_seconds: 1 availability: restart: "exit_on_failure" From 4112773773e675ad69f60fd6a06015068fa224f5 Mon Sep 17 00:00:00 2001 From: tbro Date: Tue, 4 Feb 2025 18:31:20 -0300 Subject: [PATCH 09/77] Fix path to default genesis path It appears the CLI default has drifted from real path. Also the new DEMO_GENESIS_FILE var breaks v2 demo native. So removing it for now. --- .env | 2 +- process-compose.yaml | 6 ++++-- sequencer/src/options.rs | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) mode change 100644 => 100755 sequencer/src/options.rs diff --git a/.env b/.env index d716154cfb..2beaa09a17 100644 --- a/.env +++ b/.env @@ -33,7 +33,7 @@ ESPRESSO_SEQUENCER4_API_PORT=24004 ESPRESSO_SEQUENCER_URL=http://sequencer0:${ESPRESSO_SEQUENCER_API_PORT} ESPRESSO_SEQUENCER_MAX_CONNECTIONS=25 ESPRESSO_SEQUENCER_STORAGE_PATH=/store/sequencer -ESPRESSO_SEQUENCER_GENESIS_FILE=/genesis/demo.toml +ESPRESSO_SEQUENCER_GENESIS_FILE=data/genesis/demo.toml ESPRESSO_SEQUENCER_L1_PORT=8545 ESPRESSO_SEQUENCER_L1_POLLING_INTERVAL=100ms ESPRESSO_SEQUENCER_L1_WS_PORT=8546 diff --git a/process-compose.yaml b/process-compose.yaml index 558f3d918e..450e9fd810 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -5,8 +5,8 @@ environment: - ESPRESSO_SEQUENCER_ORCHESTRATOR_URL=http://localhost:$ESPRESSO_ORCHESTRATOR_PORT - ESPRESSO_SEQUENCER_URL=http://localhost:$ESPRESSO_SEQUENCER_API_PORT - ESPRESSO_SEQUENCER_L1_PROVIDER=http://localhost:$ESPRESSO_SEQUENCER_L1_PORT - - ESPRESSO_SEQUENCER_GENESIS_FILE=$DEMO_GENESIS_FILE - - ESPRESSO_BUILDER_GENESIS_FILE=$DEMO_GENESIS_FILE + # - ESPRESSO_SEQUENCER_GENESIS_FILE=$DEMO_GENESIS_FILE + # - ESPRESSO_BUILDER_GENESIS_FILE=$DEMO_GENESIS_FILE - ESPRESSO_SEQUENCER_INITIAL_PERMISSIONED_STAKE_TABLE_PATH=data/initial_stake_table.toml - ESPRESSO_STATE_RELAY_SERVER_URL=http://localhost:$ESPRESSO_STATE_RELAY_SERVER_PORT - QUERY_SERVICE_URI=http://localhost:$ESPRESSO_SEQUENCER1_API_PORT/v0/ @@ -522,6 +522,8 @@ processes: depends_on: sequencer0: condition: process_healthy + fund-builder: + condition: process_completed readiness_probe: http_get: scheme: http diff --git a/sequencer/src/options.rs b/sequencer/src/options.rs old mode 100644 new mode 100755 index 05dbf4b39b..c818ddd401 --- a/sequencer/src/options.rs +++ b/sequencer/src/options.rs @@ -284,7 +284,7 @@ pub struct Options { long, name = "GENESIS_FILE", env = "ESPRESSO_SEQUENCER_GENESIS_FILE", - default_value = "/genesis/demo.toml" + default_value = "/data/genesis/demo.toml" )] pub genesis_file: PathBuf, From ab845844f5c566a8f7e2d259354f0111bf343d05 Mon Sep 17 00:00:00 2001 From: tbro Date: Tue, 4 Feb 2025 20:57:25 -0300 Subject: [PATCH 10/77] Some fixes Use stander env var to set genesis.file and set upgrade version to 0.3. Also make some changes to chain config to make it easier to notice if we are supplying the correct one or not. --- data/genesis/demo-pos-base.toml | 8 ++++---- justfile | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/data/genesis/demo-pos-base.toml b/data/genesis/demo-pos-base.toml index 38fb3deff5..4375fff290 100644 --- a/data/genesis/demo-pos-base.toml +++ b/data/genesis/demo-pos-base.toml @@ -1,15 +1,15 @@ base_version = "0.3" -upgrade_version = "0.0" +upgrade_version = "0.3" [stake_table] capacity = 10 [chain_config] chain_id = 999999999 -max_block_size = '1mb' -base_fee = '1 wei' +max_block_size = '2mb' +base_fee = '3 wei' fee_recipient = "0x0000000000000000000000000000000000000000" -bid_recipient = "0x0000000000000000000000000000000000000000" +# bid_recipient = "0x0000000000000000000000000000000000000000" fee_contract = "0xa15bb66138824a1c7167f5e85b957d04dd34e468" stake_table_contract = "0x8ce361602b935680e8dec218b820ff5056beb7af" diff --git a/justfile b/justfile index 1877c7eca3..1e0054356f 100644 --- a/justfile +++ b/justfile @@ -30,10 +30,10 @@ demo-native-mp *args: build scripts/demo-native -f process-compose.yaml -f process-compose-mp.yml {{args}} demo-native-pos *args: build - DEMO_GENESIS_FILE=data/genesis/demo-pos.toml scripts/demo-native -f process-compose.yaml {{args}} + ESPRESSO_SEQUENCER_GENESIS_FILE=data/genesis/demo-pos.toml scripts/demo-native -f process-compose.yaml {{args}} demo-native-pos-base *args: build - DEMO_GENESIS_FILE=data/genesis/demo-pos-base.toml scripts/demo-native -f process-compose.yaml {{args}} + ESPRESSO_SEQUENCER_GENESIS_FILE=data/genesis/demo-pos-base.toml scripts/demo-native -f process-compose.yaml {{args}} demo-native-benchmark: cargo build --release --features benchmarking From 8ed730f54a39615a18ea9ef733f5df1fffeae033 Mon Sep 17 00:00:00 2001 From: tbro Date: Tue, 4 Feb 2025 20:59:24 -0300 Subject: [PATCH 11/77] temporarily disable versions we don't need for the current objective --- sequencer/src/run.rs | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/sequencer/src/run.rs b/sequencer/src/run.rs index 7eee2e2896..7bdc0428f9 100644 --- a/sequencer/src/run.rs +++ b/sequencer/src/run.rs @@ -38,15 +38,6 @@ pub async fn main() -> anyhow::Result<()> { let upgrade = genesis.upgrade_version; match (base, upgrade) { - (FeeVersion::VERSION, EpochVersion::VERSION) => { - run( - genesis, - modules, - opt, - SequencerVersions::::new(), - ) - .await - } (EpochVersion::VERSION, _) => { run( genesis, @@ -56,15 +47,7 @@ pub async fn main() -> anyhow::Result<()> { ) .await } - (FeeVersion::VERSION, MarketplaceVersion::VERSION) => { - run( - genesis, - modules, - opt, - SequencerVersions::::new(), - ) - .await - } + _ => panic!( "Invalid base ({base}) and upgrade ({upgrade}) versions specified in the toml file." ), From 6f65adf76c0f7862a81e49798143d368b38e2058 Mon Sep 17 00:00:00 2001 From: tbro Date: Tue, 4 Feb 2025 21:00:47 -0300 Subject: [PATCH 12/77] only run v 3 in CI for now --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 645d051217..c1af831d7c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -182,13 +182,13 @@ jobs: needs: [build-test-bins, build-test-artifacts-postgres] strategy: matrix: - version: [02,03,99] + version: [03] include: - version: 02 compose: "-f process-compose.yaml -D" - version: 03 env: - DEMO_GENESIS_FILE: data/genesis/demo-pos.toml + ESPRESSO_SEQUENCER_GENESIS_FILE=data/genesis/demo-pos-base.toml compose: "-f process-compose.yaml -D" - version: 99 compose: "-f process-compose.yaml -f process-compose-mp.yml -D" From 213baf0ed4b6cbe5d95d046a41032cf0604032bb Mon Sep 17 00:00:00 2001 From: tbro Date: Tue, 4 Feb 2025 21:01:09 -0300 Subject: [PATCH 13/77] avoid redirect in header stream --- client/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/lib.rs b/client/src/lib.rs index 572d2e796e..aee2500074 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -52,7 +52,7 @@ impl SequencerClient { height: u64, ) -> anyhow::Result>> { self.0 - .socket(&format!("availability/stream/headers/{height}")) + .socket(&format!("v0/availability/stream/headers/{height}")) .subscribe::
() .await .context("subscribing to Espresso headers") From 37307e47d4ea8a00de8311963a115bd15f77f3ff Mon Sep 17 00:00:00 2001 From: tbro Date: Wed, 5 Feb 2025 11:32:39 -0300 Subject: [PATCH 14/77] Use named version (instead if integer) --- types/src/v0/impls/state.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/types/src/v0/impls/state.rs b/types/src/v0/impls/state.rs index a9a3ebc7e6..ebee2cf346 100644 --- a/types/src/v0/impls/state.rs +++ b/types/src/v0/impls/state.rs @@ -23,11 +23,11 @@ use serde::{Deserialize, Serialize}; use std::ops::Add; use thiserror::Error; use time::OffsetDateTime; -use vbs::version::Version; +use vbs::version::{StaticVersionType, Version}; use super::{ auction::ExecutionError, fee_info::FeeError, instance_state::NodeState, BlockMerkleCommitment, - BlockSize, FeeMerkleCommitment, L1Client, + BlockSize, FeeMerkleCommitment, L1Client, MarketplaceVersion, }; use crate::{ traits::StateCatchup, @@ -667,7 +667,7 @@ fn validate_builder_fee( // TODO Marketplace signatures are placeholders for now. In // finished Marketplace signatures will cover the full // transaction. - if version.minor >= 99 { + if version.minor >= MarketplaceVersion::MINOR { fee_info .account() .validate_sequencing_fee_signature_marketplace( From 3ae3f660080704a1b492ede8a540d888ec5f3ed5 Mon Sep 17 00:00:00 2001 From: tbro Date: Wed, 5 Feb 2025 14:04:41 -0300 Subject: [PATCH 15/77] Add missing header type + conversions --- types/src/v0/impls/header.rs | 4 +- types/src/v0/v0_3/chain_config.rs | 16 ++++---- types/src/v0/v0_3/header.rs | 61 ++++++++++++++++++++++++++++++ types/src/v0/v0_3/mod.rs | 17 +++++---- types/src/v0/v0_99/chain_config.rs | 53 ++++++++++++++++++++++---- 5 files changed, 126 insertions(+), 25 deletions(-) create mode 100644 types/src/v0/v0_3/header.rs diff --git a/types/src/v0/impls/header.rs b/types/src/v0/impls/header.rs index bb13ec8ab4..99189198f0 100644 --- a/types/src/v0/impls/header.rs +++ b/types/src/v0/impls/header.rs @@ -329,7 +329,7 @@ impl Header { builder_signature: builder_signature.first().copied(), }), 3 => Self::V3(v0_3::Header { - chain_config: v0_1::ResolvableChainConfig::from(v0_1::ChainConfig::from( + chain_config: v0_3::ResolvableChainConfig::from(v0_3::ChainConfig::from( chain_config, )), height, @@ -550,7 +550,7 @@ impl Header { builder_signature: builder_signature.first().copied(), }), 3 => Self::V3(v0_3::Header { - chain_config: v0_1::ResolvableChainConfig::from(v0_1::ChainConfig::from( + chain_config: v0_3::ResolvableChainConfig::from(v0_3::ChainConfig::from( chain_config, )), height, diff --git a/types/src/v0/v0_3/chain_config.rs b/types/src/v0/v0_3/chain_config.rs index dbc73cb589..f82521377c 100644 --- a/types/src/v0/v0_3/chain_config.rs +++ b/types/src/v0/v0_3/chain_config.rs @@ -1,4 +1,4 @@ -use crate::{v0_1, BlockSize, ChainId, FeeAccount, FeeAmount}; +use crate::{v0_1, v0_99, BlockSize, ChainId, FeeAccount, FeeAmount}; use committable::{Commitment, Committable}; use ethers::types::{Address, U256}; use itertools::Either; @@ -74,13 +74,13 @@ impl Committable for ChainConfig { } impl ResolvableChainConfig { - pub fn _commit(&self) -> Commitment { + pub fn commit(&self) -> Commitment { match self.chain_config { Either::Left(config) => config.commit(), Either::Right(commitment) => commitment, } } - pub fn _resolve(self) -> Option { + pub fn resolve(self) -> Option { match self.chain_config { Either::Left(config) => Some(config), Either::Right(_) => None, @@ -141,23 +141,25 @@ impl From for ChainConfig { } } -impl From for v0_1::ChainConfig { - fn from(chain_config: ChainConfig) -> v0_1::ChainConfig { - let ChainConfig { +impl From for ChainConfig { + fn from(chain_config: v0_99::ChainConfig) -> ChainConfig { + let v0_99::ChainConfig { chain_id, max_block_size, base_fee, fee_contract, fee_recipient, + stake_table_contract, .. } = chain_config; - v0_1::ChainConfig { + ChainConfig { chain_id, max_block_size, base_fee, fee_contract, fee_recipient, + stake_table_contract, } } } diff --git a/types/src/v0/v0_3/header.rs b/types/src/v0/v0_3/header.rs new file mode 100644 index 0000000000..c4dd120916 --- /dev/null +++ b/types/src/v0/v0_3/header.rs @@ -0,0 +1,61 @@ +use crate::NsTable; + +use super::{ + BlockMerkleCommitment, BuilderSignature, FeeInfo, FeeMerkleCommitment, L1BlockInfo, + ResolvableChainConfig, +}; +use ark_serialize::CanonicalSerialize; +use committable::{Commitment, Committable, RawCommitmentBuilder}; +use hotshot_types::{utils::BuilderCommitment, vid::VidCommitment}; +use serde::{Deserialize, Serialize}; + +/// A header is like a [`Block`] with the body replaced by a digest. +#[derive(Clone, Debug, Deserialize, Serialize, Hash, PartialEq, Eq)] +pub struct Header { + /// A commitment to a ChainConfig or a full ChainConfig. + pub(crate) chain_config: ResolvableChainConfig, + pub(crate) height: u64, + pub(crate) timestamp: u64, + pub(crate) l1_head: u64, + pub(crate) l1_finalized: Option, + pub(crate) payload_commitment: VidCommitment, + pub(crate) builder_commitment: BuilderCommitment, + pub(crate) ns_table: NsTable, + pub(crate) block_merkle_tree_root: BlockMerkleCommitment, + pub(crate) fee_merkle_tree_root: FeeMerkleCommitment, + pub(crate) fee_info: FeeInfo, + pub(crate) builder_signature: Option, +} + +impl Committable for Header { + fn commit(&self) -> Commitment { + let mut bmt_bytes = vec![]; + self.block_merkle_tree_root + .serialize_with_mode(&mut bmt_bytes, ark_serialize::Compress::Yes) + .unwrap(); + let mut fmt_bytes = vec![]; + self.fee_merkle_tree_root + .serialize_with_mode(&mut fmt_bytes, ark_serialize::Compress::Yes) + .unwrap(); + + RawCommitmentBuilder::new(&Self::tag()) + .field("chain_config", self.chain_config.commit()) + .u64_field("height", self.height) + .u64_field("timestamp", self.timestamp) + .u64_field("l1_head", self.l1_head) + .optional("l1_finalized", &self.l1_finalized) + .constant_str("payload_commitment") + .fixed_size_bytes(self.payload_commitment.as_ref().as_ref()) + .constant_str("builder_commitment") + .fixed_size_bytes(self.builder_commitment.as_ref()) + .field("ns_table", self.ns_table.commit()) + .var_size_field("block_merkle_tree_root", &bmt_bytes) + .var_size_field("fee_merkle_tree_root", &fmt_bytes) + .field("fee_info", self.fee_info.commit()) + .finalize() + } + + fn tag() -> String { + crate::v0_1::Header::tag() + } +} diff --git a/types/src/v0/v0_3/mod.rs b/types/src/v0/v0_3/mod.rs index 23c8e3a021..9c7789d1e9 100644 --- a/types/src/v0/v0_3/mod.rs +++ b/types/src/v0/v0_3/mod.rs @@ -4,14 +4,13 @@ use vbs::version::Version; pub use super::v0_1::{ AccountQueryData, BlockMerkleCommitment, BlockMerkleTree, BlockSize, BuilderSignature, ChainId, Delta, FeeAccount, FeeAccountProof, FeeAmount, FeeInfo, FeeMerkleCommitment, FeeMerkleProof, - FeeMerkleTree, Header, Index, Iter, L1BlockInfo, L1Client, L1ClientOptions, L1Snapshot, - NamespaceId, NsIndex, NsIter, NsPayload, NsPayloadBuilder, NsPayloadByteLen, NsPayloadOwned, - NsPayloadRange, NsProof, NsTable, NsTableBuilder, NsTableValidationError, NumNss, NumTxs, - NumTxsRange, NumTxsUnchecked, Payload, PayloadByteLen, ResolvableChainConfig, TimeBasedUpgrade, - Transaction, TxIndex, TxIter, TxPayload, TxPayloadRange, TxProof, TxTableEntries, - TxTableEntriesRange, Upgrade, UpgradeMode, UpgradeType, ViewBasedUpgrade, - BLOCK_MERKLE_TREE_HEIGHT, FEE_MERKLE_TREE_HEIGHT, NS_ID_BYTE_LEN, NS_OFFSET_BYTE_LEN, - NUM_NSS_BYTE_LEN, NUM_TXS_BYTE_LEN, TX_OFFSET_BYTE_LEN, + FeeMerkleTree, Index, Iter, L1BlockInfo, L1Client, L1ClientOptions, L1Snapshot, NamespaceId, + NsIndex, NsIter, NsPayload, NsPayloadBuilder, NsPayloadByteLen, NsPayloadOwned, NsPayloadRange, + NsProof, NsTable, NsTableBuilder, NsTableValidationError, NumNss, NumTxs, NumTxsRange, + NumTxsUnchecked, Payload, PayloadByteLen, TimeBasedUpgrade, Transaction, TxIndex, TxIter, + TxPayload, TxPayloadRange, TxProof, TxTableEntries, TxTableEntriesRange, Upgrade, UpgradeMode, + UpgradeType, ViewBasedUpgrade, BLOCK_MERKLE_TREE_HEIGHT, FEE_MERKLE_TREE_HEIGHT, + NS_ID_BYTE_LEN, NS_OFFSET_BYTE_LEN, NUM_NSS_BYTE_LEN, NUM_TXS_BYTE_LEN, TX_OFFSET_BYTE_LEN, }; pub(crate) use super::v0_1::{ L1ClientMetrics, L1Event, L1Provider, L1State, L1UpdateTask, MultiRpcClient, @@ -21,7 +20,9 @@ pub(crate) use super::v0_1::{ pub const VERSION: Version = Version { major: 0, minor: 3 }; mod chain_config; +mod header; mod stake_table; pub use chain_config::*; +pub use header::Header; pub use stake_table::*; diff --git a/types/src/v0/v0_99/chain_config.rs b/types/src/v0/v0_99/chain_config.rs index 420580d7af..cd9fd7cba5 100644 --- a/types/src/v0/v0_99/chain_config.rs +++ b/types/src/v0/v0_99/chain_config.rs @@ -130,6 +130,21 @@ impl From<&v0_1::ResolvableChainConfig> for ResolvableChainConfig { } } +impl From<&v0_3::ResolvableChainConfig> for ResolvableChainConfig { + fn from( + &v0_3::ResolvableChainConfig { chain_config }: &v0_3::ResolvableChainConfig, + ) -> ResolvableChainConfig { + match chain_config { + Either::Left(chain_config) => ResolvableChainConfig { + chain_config: Either::Left(ChainConfig::from(chain_config)), + }, + Either::Right(c) => ResolvableChainConfig { + chain_config: Either::Right(Commitment::from_raw(*c.as_ref())), + }, + } + } +} + impl From for ChainConfig { fn from(chain_config: v0_1::ChainConfig) -> ChainConfig { let v0_1::ChainConfig { @@ -217,22 +232,44 @@ mod test { use super::*; #[test] - fn test_upgrade_chain_config_v3_resolvable_chain_config_from_v1() { + fn test_upgrade_chain_config_v99_resolvable_chain_config_from_v1() { let expectation: ResolvableChainConfig = ChainConfig::default().into(); let v1_resolvable: v0_1::ResolvableChainConfig = v0_1::ChainConfig::default().into(); - let v3_resolvable: ResolvableChainConfig = ResolvableChainConfig::from(&v1_resolvable); - assert_eq!(expectation, v3_resolvable); + let v99_resolvable: ResolvableChainConfig = ResolvableChainConfig::from(&v1_resolvable); + assert_eq!(expectation, v99_resolvable); let expectation: ResolvableChainConfig = ChainConfig::default().commit().into(); let v1_resolvable: v0_1::ResolvableChainConfig = v0_1::ChainConfig::default().commit().into(); - let v3_resolvable: ResolvableChainConfig = ResolvableChainConfig::from(&v1_resolvable); - assert_eq!(expectation, v3_resolvable); + let v99_resolvable: ResolvableChainConfig = ResolvableChainConfig::from(&v1_resolvable); + assert_eq!(expectation, v99_resolvable); } + #[test] - fn test_upgrade_chain_config_v1_chain_config_from_v3() { + fn test_upgrade_chain_config_v99_resolvable_chain_config_from_v3() { + let expectation: ResolvableChainConfig = ChainConfig::default().into(); + let v3_resolvable: v0_3::ResolvableChainConfig = v0_3::ChainConfig::default().into(); + let v99_resolvable: ResolvableChainConfig = ResolvableChainConfig::from(&v3_resolvable); + assert_eq!(expectation, v99_resolvable); + let expectation: ResolvableChainConfig = ChainConfig::default().commit().into(); + let v3_resolvable: v0_3::ResolvableChainConfig = + v0_3::ChainConfig::default().commit().into(); + let v99_resolvable: ResolvableChainConfig = ResolvableChainConfig::from(&v3_resolvable); + assert_eq!(expectation, v99_resolvable); + } + + #[test] + fn test_upgrade_chain_config_v1_chain_config_from_v99() { let expectation = v0_1::ChainConfig::default(); - let v3_chain_config = ChainConfig::default(); - let v1_chain_config = v0_1::ChainConfig::from(v3_chain_config); + let v99_chain_config = ChainConfig::default(); + let v1_chain_config = v0_1::ChainConfig::from(v99_chain_config); assert_eq!(expectation, v1_chain_config); } + + #[test] + fn test_upgrade_chain_config_v3_chain_config_from_v99() { + let expectation = v0_3::ChainConfig::default(); + let v99_chain_config = ChainConfig::default(); + let v3_chain_config = v0_3::ChainConfig::from(v99_chain_config); + assert_eq!(expectation, v3_chain_config); + } } From 7c6222b441563c565036a7bfb53696d4993012bc Mon Sep 17 00:00:00 2001 From: tbro Date: Wed, 5 Feb 2025 14:19:33 -0300 Subject: [PATCH 16/77] bump todo_by by 1 month (#2534) The date has passed again leading to warnings/errors during compilation. --- sequencer/src/network/cdn.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sequencer/src/network/cdn.rs b/sequencer/src/network/cdn.rs index cc524b026a..adce13a758 100644 --- a/sequencer/src/network/cdn.rs +++ b/sequencer/src/network/cdn.rs @@ -80,7 +80,7 @@ impl SignatureScheme for WrappedSignatureKey { }; todo_by!( - "2025-2-4", + "2025-3-4", "Only accept the namespaced message once everyone has upgraded" ); public_key.0.validate(&signature, message) @@ -112,7 +112,7 @@ impl RunDef for ProductionDef { } todo_by!( - "2025-2-4", + "2025-3-4", "Remove this, switching to TCP+TLS singularly when everyone has updated" ); /// The user definition for the Push CDN. From a65cbb82e28f89fe00b328d245661821b1469082 Mon Sep 17 00:00:00 2001 From: tbro Date: Wed, 5 Feb 2025 14:39:43 -0300 Subject: [PATCH 17/77] Put expected fee back as it was --- data/genesis/demo-pos-base.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/genesis/demo-pos-base.toml b/data/genesis/demo-pos-base.toml index 4375fff290..b7a5994ced 100644 --- a/data/genesis/demo-pos-base.toml +++ b/data/genesis/demo-pos-base.toml @@ -6,8 +6,8 @@ capacity = 10 [chain_config] chain_id = 999999999 -max_block_size = '2mb' -base_fee = '3 wei' +max_block_size = '1mb' +base_fee = '1 wei' fee_recipient = "0x0000000000000000000000000000000000000000" # bid_recipient = "0x0000000000000000000000000000000000000000" fee_contract = "0xa15bb66138824a1c7167f5e85b957d04dd34e468" From 9613ce1b242c2436bfbb21b0b2d8f359483f7267 Mon Sep 17 00:00:00 2001 From: tbro Date: Wed, 5 Feb 2025 17:58:49 -0300 Subject: [PATCH 18/77] Add stake tables up to epoch 2 (inclusive) --- types/src/v0/impls/stake_table.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/types/src/v0/impls/stake_table.rs b/types/src/v0/impls/stake_table.rs index 9b62a09515..81acfcb0ad 100644 --- a/types/src/v0/impls/stake_table.rs +++ b/types/src/v0/impls/stake_table.rs @@ -240,10 +240,11 @@ impl EpochCommittees { indexed_da_members, }; + // TODO: remove this, workaround for hotshot asking for stake tables from epoch 1 and 2 let mut map = HashMap::new(); - map.insert(Epoch::genesis(), members.clone()); - // TODO: remove this, workaround for hotshot asking for stake tables from epoch 1 - map.insert(Epoch::genesis() + 1u64, members.clone()); + for epoch in Epoch::genesis().u64()..=2 { + map.insert(Epoch::new(epoch), members.clone()); + } Self { non_epoch_committee: members, From cbb414c229918d127cfbdd17266a58114cf68905 Mon Sep 17 00:00:00 2001 From: tbro Date: Wed, 5 Feb 2025 18:14:45 -0300 Subject: [PATCH 19/77] add stake table for ..=10 --- types/src/v0/impls/stake_table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/src/v0/impls/stake_table.rs b/types/src/v0/impls/stake_table.rs index 81acfcb0ad..802e59af9c 100644 --- a/types/src/v0/impls/stake_table.rs +++ b/types/src/v0/impls/stake_table.rs @@ -242,7 +242,7 @@ impl EpochCommittees { // TODO: remove this, workaround for hotshot asking for stake tables from epoch 1 and 2 let mut map = HashMap::new(); - for epoch in Epoch::genesis().u64()..=2 { + for epoch in Epoch::genesis().u64()..=10 { map.insert(Epoch::new(epoch), members.clone()); } From aa31ce96e6257fba6e925acb7fbac72a4e7fbdf4 Mon Sep 17 00:00:00 2001 From: tbro Date: Thu, 6 Feb 2025 16:40:56 -0300 Subject: [PATCH 20/77] TODO questions --- sequencer/src/genesis.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sequencer/src/genesis.rs b/sequencer/src/genesis.rs index aa247ce614..0b5d5e3526 100644 --- a/sequencer/src/genesis.rs +++ b/sequencer/src/genesis.rs @@ -83,6 +83,7 @@ impl Genesis { } impl Genesis { + // TODO `validate_stake_table_contract` and wrapper `validate_contracts` pub async fn validate_fee_contract(&self, l1_rpc_url: Url) -> anyhow::Result<()> { let l1 = L1Client::new(l1_rpc_url); @@ -100,7 +101,8 @@ impl Genesis { // now iterate over each upgrade type and validate the fee contract if it exists for (version, upgrade) in &self.upgrades { let chain_config = &upgrade.upgrade_type.chain_config(); - + // Is this not an error case? Isn't a chain config a + // requirement? At least for most versions? if chain_config.is_none() { continue; } From a4e0b58c159bfc2e11f6047f78b5032b457a7482 Mon Sep 17 00:00:00 2001 From: tbro Date: Thu, 6 Feb 2025 16:41:12 -0300 Subject: [PATCH 21/77] add backtrace to justfile disabled --- justfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/justfile b/justfile index 1e0054356f..a8f72c972b 100644 --- a/justfile +++ b/justfile @@ -1,3 +1,5 @@ +# export RUST_BACKTRACE := "1" + default: just --list From 7203717a7d4b97c819aa18a26fa8e844bdef5cb0 Mon Sep 17 00:00:00 2001 From: tbro Date: Thu, 6 Feb 2025 16:41:51 -0300 Subject: [PATCH 22/77] TODO --- sequencer/src/bin/espresso-bridge.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sequencer/src/bin/espresso-bridge.rs b/sequencer/src/bin/espresso-bridge.rs index 2ae2c02632..e822aff838 100644 --- a/sequencer/src/bin/espresso-bridge.rs +++ b/sequencer/src/bin/espresso-bridge.rs @@ -231,6 +231,8 @@ async fn deposit(opt: Deposit) -> anyhow::Result<()> { } }; + // TODO this appears to be broken. We often hit the `else` block + // when the builder was in fact funded. // Confirm that the Espresso balance has increased. let final_balance = espresso .get_espresso_balance(l1.address(), Some(espresso_block)) From 2f28e000dcdf864c9aad4bbe76523b431a7835e1 Mon Sep 17 00:00:00 2001 From: tbro Date: Thu, 6 Feb 2025 16:42:00 -0300 Subject: [PATCH 23/77] comment --- sequencer/src/run.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sequencer/src/run.rs b/sequencer/src/run.rs index 7bdc0428f9..be4cfc61f2 100644 --- a/sequencer/src/run.rs +++ b/sequencer/src/run.rs @@ -43,6 +43,7 @@ pub async fn main() -> anyhow::Result<()> { genesis, modules, opt, + // Specifying V0_0 disables upgrades SequencerVersions::::new(), ) .await From 98a6969d2c3feb6fa26eae13e49c7f70753851a1 Mon Sep 17 00:00:00 2001 From: tbro Date: Fri, 7 Feb 2025 15:44:59 -0300 Subject: [PATCH 24/77] version features --- Cargo.toml | 16 ---------------- sequencer/Cargo.toml | 4 ++++ sequencer/src/run.rs | 22 +++++++++++++++++++++- tests/smoke.rs | 1 + 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c562240ca3..d4a38723f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -180,19 +180,3 @@ paste = "1.0" rand = "0.8.5" time = "0.3" trait-set = "0.3.0" - -[profile.dev] -# No optimizations -opt-level = 0 -# Skip compiling the debug information. -debug = false -# Skip linking symbols. -strip = true -[profile.test] -opt-level = 1 -[profile.test.package.tests] -opt-level = 0 -[profile.test.package.client] -opt-level = 0 -[profile.test.package.hotshot-state-prover] -opt-level = 3 diff --git a/sequencer/Cargo.toml b/sequencer/Cargo.toml index 65d1a1b49b..3a49f31185 100644 --- a/sequencer/Cargo.toml +++ b/sequencer/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] +default = ["pos"] testing = [ "hotshot-testing", "marketplace-builder-core", @@ -15,6 +16,9 @@ testing = [ ] benchmarking = [] embedded-db = ["hotshot-query-service/embedded-db"] +fee = [] +pos = [] +marketplace = [] [[bin]] name = "espresso-dev-node" diff --git a/sequencer/src/run.rs b/sequencer/src/run.rs index c8862b1a8b..d6efcf2314 100644 --- a/sequencer/src/run.rs +++ b/sequencer/src/run.rs @@ -38,6 +38,17 @@ pub async fn main() -> anyhow::Result<()> { let upgrade = genesis.upgrade_version; match (base, upgrade) { + #[cfg(all(feature = "fee", feature = "pos"))] + (FeeVersion::VERSION, EpochVersion::VERSION) => { + run( + genesis, + modules, + opt, + SequencerVersions::::new(), + ) + .await + } + #[cfg(feature = "pos")] (EpochVersion::VERSION, _) => { run( genesis, @@ -48,7 +59,16 @@ pub async fn main() -> anyhow::Result<()> { ) .await } - + #[cfg(all(feature = "fee", feature = "marketplace"))] + (FeeVersion::VERSION, MarketplaceVersion::VERSION) => { + run( + genesis, + modules, + opt, + SequencerVersions::::new(), + ) + .await + } _ => panic!( "Invalid base ({base}) and upgrade ({upgrade}) versions specified in the toml file." ), diff --git a/tests/smoke.rs b/tests/smoke.rs index d154fc03dd..69ea3be272 100644 --- a/tests/smoke.rs +++ b/tests/smoke.rs @@ -30,6 +30,7 @@ async fn test_smoke() -> Result<()> { let mut state_retries = 0; let mut txn_retries = 0; while (sub.next().await).is_some() { + dbg!("next"); let new = testing.test_state().await; println!("New State:{}", new); From 711c7b9300df4280edc38a416b65b68db9554f21 Mon Sep 17 00:00:00 2001 From: tbro Date: Fri, 7 Feb 2025 17:30:33 -0300 Subject: [PATCH 25/77] I think build jet has more space (lets find out) --- .github/workflows/cargo-features.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cargo-features.yml b/.github/workflows/cargo-features.yml index 572cbd24c0..2b39663a93 100644 --- a/.github/workflows/cargo-features.yml +++ b/.github/workflows/cargo-features.yml @@ -19,7 +19,7 @@ concurrency: jobs: cargo-features: - runs-on: ubuntu-latest + runs-on: buildjet-8vcpu-ubuntu-2204 steps: - uses: taiki-e/install-action@cargo-hack From dc25ada5892fb3254ac2343a9e164874b844e8e4 Mon Sep 17 00:00:00 2001 From: tbro Date: Fri, 7 Feb 2025 17:32:52 -0300 Subject: [PATCH 26/77] Revert "I think build jet has more space (lets find out)" This reverts commit 711c7b9300df4280edc38a416b65b68db9554f21. --- .github/workflows/cargo-features.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cargo-features.yml b/.github/workflows/cargo-features.yml index 2b39663a93..572cbd24c0 100644 --- a/.github/workflows/cargo-features.yml +++ b/.github/workflows/cargo-features.yml @@ -19,7 +19,7 @@ concurrency: jobs: cargo-features: - runs-on: buildjet-8vcpu-ubuntu-2204 + runs-on: ubuntu-latest steps: - uses: taiki-e/install-action@cargo-hack From e951d1d3a1769f0cb628c53be878384c02e91145 Mon Sep 17 00:00:00 2001 From: tbro Date: Fri, 7 Feb 2025 18:36:57 -0300 Subject: [PATCH 27/77] test epoch test --- sequencer/src/api.rs | 87 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 5165b9f5c7..063822a16f 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -1605,6 +1605,7 @@ mod test { persistence::no_storage, testing::{TestConfig, TestConfigBuilder}, }; + use espresso_types::EpochVersion; #[tokio::test(flavor = "multi_thread")] async fn test_healthcheck() { @@ -2246,6 +2247,36 @@ mod test { test_upgrade_helper::(upgrades, MySequencerVersions::new()).await; } + #[tokio::test(flavor = "multi_thread")] + async fn test_pos_upgrade_view_based() { + setup_test(); + + let mut upgrades = std::collections::BTreeMap::new(); + type MySequencerVersions = SequencerVersions; + + let mode = UpgradeMode::View(ViewBasedUpgrade { + start_voting_view: None, + stop_voting_view: None, + start_proposing_view: 1, + stop_proposing_view: 10, + }); + + let upgrade_type = UpgradeType::Marketplace { + chain_config: ChainConfig { + max_block_size: 400.into(), + base_fee: 2.into(), + bid_recipient: Some(Default::default()), + ..Default::default() + }, + }; + + upgrades.insert( + ::Upgrade::VERSION, + Upgrade { mode, upgrade_type }, + ); + test_upgrade_helper::(upgrades, MySequencerVersions::new()).await; + } + async fn test_upgrade_helper( upgrades: BTreeMap, bind_version: MockSeqVersions, @@ -2570,6 +2601,62 @@ mod test { let mut receive_count = 0; loop { let event = subscribed_events.next().await.unwrap(); + dbg!(&event); + tracing::info!( + "Received event in hotshot event streaming Client 1: {:?}", + event + ); + receive_count += 1; + if receive_count > total_count { + tracing::info!("Client Received at least desired events, exiting loop"); + break; + } + } + assert_eq!(receive_count, total_count + 1); + } + // TODO instead of as above, listen to events until we get at least to view 3 + // maybe put in the slow category + #[tokio::test(flavor = "multi_thread")] + async fn test_hotshot_event_streaming_epoch_progression() { + setup_test(); + + let hotshot_event_streaming_port = + pick_unused_port().expect("No ports free for hotshot event streaming"); + let query_service_port = pick_unused_port().expect("No ports free for query service"); + + let url = format!("http://localhost:{hotshot_event_streaming_port}") + .parse() + .unwrap(); + + let hotshot_events = HotshotEvents { + events_service_port: hotshot_event_streaming_port, + }; + + let client: Client = Client::new(url); + + let options = Options::with_port(query_service_port).hotshot_events(hotshot_events); + + let anvil = Anvil::new().spawn(); + let l1 = anvil.endpoint().parse().unwrap(); + let network_config = TestConfigBuilder::default().l1_url(l1).build(); + let config = TestNetworkConfigBuilder::default() + .api_config(options) + .network_config(network_config) + .build(); + let _network = TestNetwork::new(config, MockSequencerVersions::new()).await; + + let mut subscribed_events = client + .socket("hotshot-events/events") + .subscribe::>() + .await + .unwrap(); + + let total_count = 5; + // wait for these events to receive on client 1 + let mut receive_count = 0; + loop { + let event = subscribed_events.next().await.unwrap(); + dbg!(&event); tracing::info!( "Received event in hotshot event streaming Client 1: {:?}", event From 64eb0d848b9f8f4d8e167c08bcbd96062fe7e9c8 Mon Sep 17 00:00:00 2001 From: tbro Date: Mon, 10 Feb 2025 17:46:29 -0300 Subject: [PATCH 28/77] Make epoch_height on TestNetwork configurable Also adds an (incomplete) test to check that views are progressing beyond epoch boundries --- sequencer/src/api.rs | 54 +++++++++++++++++++++++++++++++++----------- sequencer/src/lib.rs | 5 ++++ 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 063822a16f..4836247f4b 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -1557,7 +1557,10 @@ mod api_tests { #[cfg(test)] mod test { use committable::{Commitment, Committable}; - use std::{collections::BTreeMap, time::Duration}; + use std::{ + collections::{BTreeMap, HashSet}, + time::Duration, + }; use tokio::time::sleep; use espresso_types::{ @@ -2614,12 +2617,16 @@ mod test { } assert_eq!(receive_count, total_count + 1); } - // TODO instead of as above, listen to events until we get at least to view 3 - // maybe put in the slow category + // TODO unfinished test. the idea is to observe epochs and views + // are progressing in a sane way #[tokio::test(flavor = "multi_thread")] async fn test_hotshot_event_streaming_epoch_progression() { setup_test(); + // TODO currently getting `hotshot_task_impls::helpers: : Failed epoch safety check` + // at epoch_height + 1 + let epoch_height = 5; + let hotshot_event_streaming_port = pick_unused_port().expect("No ports free for hotshot event streaming"); let query_service_port = pick_unused_port().expect("No ports free for query service"); @@ -2638,7 +2645,10 @@ mod test { let anvil = Anvil::new().spawn(); let l1 = anvil.endpoint().parse().unwrap(); - let network_config = TestConfigBuilder::default().l1_url(l1).build(); + let network_config = TestConfigBuilder::default() + .l1_url(l1) + .with_epoch_height(epoch_height) + .build(); let config = TestNetworkConfigBuilder::default() .api_config(options) .network_config(network_config) @@ -2651,22 +2661,40 @@ mod test { .await .unwrap(); - let total_count = 5; + // wanted views + let total_count = epoch_height * 2; // wait for these events to receive on client 1 let mut receive_count = 0; + let mut views = HashSet::new(); + let mut i = 0; loop { let event = subscribed_events.next().await.unwrap(); - dbg!(&event); - tracing::info!( - "Received event in hotshot event streaming Client 1: {:?}", - event - ); - receive_count += 1; - if receive_count > total_count { - tracing::info!("Client Received at least desired events, exiting loop"); + let event = event.unwrap(); + let view_number = event.view_number; + views.insert(view_number); + dbg!(view_number); + + if let hotshot::types::EventType::Decide { .. } = event.event { + dbg!("got decide"); + + receive_count += 1; + } + // dbg!(event.clone().unwrap().view_number); + // tracing::info!( + // "Received event in hotshot event streaming Client 1: {:?}", + // event + // ); + if views.contains(&ViewNumber::new(total_count)) { + tracing::info!("Client Received at least desired views, exiting loop"); break; } + if i > 100 { + // Timeout + panic!("Views are not progressing"); + } + i += 1; } + // TODO this is still just a place holder assert_eq!(receive_count, total_count + 1); } } diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index f63a00f42a..e394d48fdf 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -741,6 +741,11 @@ pub mod testing { self } + pub fn with_epoch_height(mut self, epoch_height: u64) -> Self { + self.config.epoch_height = epoch_height; + self + } + pub fn upgrades(mut self, upgrades: BTreeMap) -> Self { let upgrade = upgrades.get(&::Upgrade::VERSION).unwrap(); upgrade.set_hotshot_config_parameters(&mut self.config); From c9eeb2af57671b9f22eca89d8d5eefa46fac830d Mon Sep 17 00:00:00 2001 From: tbro Date: Mon, 10 Feb 2025 19:08:35 -0300 Subject: [PATCH 29/77] fix --- sequencer/src/api.rs | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 4836247f4b..4be855bf5e 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -2626,6 +2626,7 @@ mod test { // TODO currently getting `hotshot_task_impls::helpers: : Failed epoch safety check` // at epoch_height + 1 let epoch_height = 5; + type PosVersion = SequencerVersions, StaticVersion<0, 0>>; let hotshot_event_streaming_port = pick_unused_port().expect("No ports free for hotshot event streaming"); @@ -2653,7 +2654,7 @@ mod test { .api_config(options) .network_config(network_config) .build(); - let _network = TestNetwork::new(config, MockSequencerVersions::new()).await; + let _network = TestNetwork::new(config, PosVersion::new()).await; let mut subscribed_events = client .socket("hotshot-events/events") @@ -2664,27 +2665,18 @@ mod test { // wanted views let total_count = epoch_height * 2; // wait for these events to receive on client 1 - let mut receive_count = 0; let mut views = HashSet::new(); let mut i = 0; loop { let event = subscribed_events.next().await.unwrap(); let event = event.unwrap(); let view_number = event.view_number; - views.insert(view_number); - dbg!(view_number); + views.insert(view_number.u64()); if let hotshot::types::EventType::Decide { .. } = event.event { dbg!("got decide"); - - receive_count += 1; } - // dbg!(event.clone().unwrap().view_number); - // tracing::info!( - // "Received event in hotshot event streaming Client 1: {:?}", - // event - // ); - if views.contains(&ViewNumber::new(total_count)) { + if views.contains(&total_count) { tracing::info!("Client Received at least desired views, exiting loop"); break; } @@ -2694,7 +2686,6 @@ mod test { } i += 1; } - // TODO this is still just a place holder - assert_eq!(receive_count, total_count + 1); + assert!(views.contains(&total_count)); } } From 87b15471fcee5a752b02dd56a0e49d3cbbdcb65c Mon Sep 17 00:00:00 2001 From: tbro Date: Mon, 10 Feb 2025 19:16:57 -0300 Subject: [PATCH 30/77] remove comment --- sequencer/src/api.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 4be855bf5e..083ed2fa0f 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -2623,8 +2623,6 @@ mod test { async fn test_hotshot_event_streaming_epoch_progression() { setup_test(); - // TODO currently getting `hotshot_task_impls::helpers: : Failed epoch safety check` - // at epoch_height + 1 let epoch_height = 5; type PosVersion = SequencerVersions, StaticVersion<0, 0>>; From 86fd4a90cec70668129d2f7ab90c79d0f38528e6 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 11 Feb 2025 04:12:24 +0500 Subject: [PATCH 31/77] lock file --- Cargo.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1772d8d2d5..f112b09191 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1806,7 +1806,7 @@ dependencies = [ "bitflags 2.6.0", "cexpr", "clang-sys", - "itertools 0.11.0", + "itertools 0.10.5", "log", "prettyplease", "proc-macro2", @@ -2483,9 +2483,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" dependencies = [ "cc", ] @@ -5591,7 +5591,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite 0.2.16", - "socket2 0.4.10", + "socket2 0.5.8", "tokio", "tower-service", "tracing", @@ -6593,7 +6593,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -8663,7 +8663,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.95", @@ -10326,7 +10326,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.95", From 1d47a0806563ba9e371caf41716c9fd0258acfc0 Mon Sep 17 00:00:00 2001 From: tbro Date: Tue, 11 Feb 2025 11:58:24 -0300 Subject: [PATCH 32/77] add test profile --- Cargo.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 8006e9045d..94308888ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -224,3 +224,11 @@ paste = "1.0" rand = "0.8.5" time = "0.3" trait-set = "0.3.0" +[profile.test] +opt-level = 1 +[profile.test.package.tests] +opt-level = 0 +[profile.test.package.client] +opt-level = 0 +[profile.test.package.hotshot-state-prover] +opt-level = 3 From 9936c1ee5be92145c3edf38754478a7bf3d9a7de Mon Sep 17 00:00:00 2001 From: tbro Date: Wed, 12 Feb 2025 16:39:52 -0300 Subject: [PATCH 33/77] Add pos view based upgrade test Currently does not pass. We get votes but after voting there is not progress. Removed fee upgrade test, b/c there is no longer a version earlier than fee. --- sequencer/src/api.rs | 76 +++++--------------------------------------- sequencer/src/run.rs | 1 + 2 files changed, 9 insertions(+), 68 deletions(-) diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 761b253d2e..ec9cca931d 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -2130,11 +2130,11 @@ mod test { } #[tokio::test(flavor = "multi_thread")] - async fn test_fee_upgrade_view_based() { + async fn test_pos_upgrade_view_based() { setup_test(); let mut upgrades = std::collections::BTreeMap::new(); - type MySequencerVersions = SequencerVersions, StaticVersion<0, 2>>; + type MySequencerVersions = SequencerVersions; let mode = UpgradeMode::View(ViewBasedUpgrade { start_voting_view: None, @@ -2143,41 +2143,11 @@ mod test { stop_proposing_view: 10, }); - let upgrade_type = UpgradeType::Fee { - chain_config: ChainConfig { - max_block_size: 300.into(), - base_fee: 1.into(), - ..Default::default() - }, - }; - - upgrades.insert( - ::Upgrade::VERSION, - Upgrade { mode, upgrade_type }, - ); - test_upgrade_helper::(upgrades, MySequencerVersions::new()).await; - } - - #[tokio::test(flavor = "multi_thread")] - async fn test_fee_upgrade_time_based() { - setup_test(); - - let now = OffsetDateTime::now_utc().unix_timestamp() as u64; - - let mut upgrades = std::collections::BTreeMap::new(); - type MySequencerVersions = SequencerVersions, StaticVersion<0, 2>>; - - let mode = UpgradeMode::Time(TimeBasedUpgrade { - start_proposing_time: Timestamp::from_integer(now).unwrap(), - stop_proposing_time: Timestamp::from_integer(now + 500).unwrap(), - start_voting_time: None, - stop_voting_time: None, - }); - - let upgrade_type = UpgradeType::Fee { + let upgrade_type = UpgradeType::Epoch { chain_config: ChainConfig { - max_block_size: 300.into(), - base_fee: 1.into(), + max_block_size: 500.into(), + base_fee: 2.into(), + stake_table_contract: Some(Default::default()), ..Default::default() }, }; @@ -2194,7 +2164,7 @@ mod test { setup_test(); let mut upgrades = std::collections::BTreeMap::new(); - type MySequencerVersions = SequencerVersions; + type MySequencerVersions = SequencerVersions; let mode = UpgradeMode::View(ViewBasedUpgrade { start_voting_view: None, @@ -2226,7 +2196,7 @@ mod test { let now = OffsetDateTime::now_utc().unix_timestamp() as u64; let mut upgrades = std::collections::BTreeMap::new(); - type MySequencerVersions = SequencerVersions; + type MySequencerVersions = SequencerVersions; let mode = UpgradeMode::Time(TimeBasedUpgrade { start_proposing_time: Timestamp::from_integer(now).unwrap(), @@ -2251,36 +2221,6 @@ mod test { test_upgrade_helper::(upgrades, MySequencerVersions::new()).await; } - #[tokio::test(flavor = "multi_thread")] - async fn test_pos_upgrade_view_based() { - setup_test(); - - let mut upgrades = std::collections::BTreeMap::new(); - type MySequencerVersions = SequencerVersions; - - let mode = UpgradeMode::View(ViewBasedUpgrade { - start_voting_view: None, - stop_voting_view: None, - start_proposing_view: 1, - stop_proposing_view: 10, - }); - - let upgrade_type = UpgradeType::Marketplace { - chain_config: ChainConfig { - max_block_size: 400.into(), - base_fee: 2.into(), - bid_recipient: Some(Default::default()), - ..Default::default() - }, - }; - - upgrades.insert( - ::Upgrade::VERSION, - Upgrade { mode, upgrade_type }, - ); - test_upgrade_helper::(upgrades, MySequencerVersions::new()).await; - } - async fn test_upgrade_helper( upgrades: BTreeMap, bind_version: MockSeqVersions, diff --git a/sequencer/src/run.rs b/sequencer/src/run.rs index d6efcf2314..6efe90c484 100644 --- a/sequencer/src/run.rs +++ b/sequencer/src/run.rs @@ -59,6 +59,7 @@ pub async fn main() -> anyhow::Result<()> { ) .await } + // TODO change `fee` to `pos` #[cfg(all(feature = "fee", feature = "marketplace"))] (FeeVersion::VERSION, MarketplaceVersion::VERSION) => { run( From 0d91dc86c5bcaf9d494e7aa4aaad019758df1413 Mon Sep 17 00:00:00 2001 From: tbro Date: Thu, 13 Feb 2025 15:12:37 -0300 Subject: [PATCH 34/77] Fix chain_config upgrade in Header::new --- types/src/v0/impls/header.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/types/src/v0/impls/header.rs b/types/src/v0/impls/header.rs index 3db2309edb..427dd2bf21 100644 --- a/types/src/v0/impls/header.rs +++ b/types/src/v0/impls/header.rs @@ -978,6 +978,7 @@ impl BlockHeader for Header { match instance_state.upgrades.get(&version) { Some(upgrade) => match upgrade.upgrade_type { UpgradeType::Fee { chain_config } => chain_config, + UpgradeType::Epoch { chain_config } => chain_config, _ => Header::get_chain_config(&validated_state, instance_state).await?, }, None => Header::get_chain_config(&validated_state, instance_state).await?, From 0965d904a0437a5393bc4ea00d4a196b78d9e595 Mon Sep 17 00:00:00 2001 From: tbro Date: Thu, 13 Feb 2025 15:13:06 -0300 Subject: [PATCH 35/77] remove a TODO (hurray!) --- sequencer/src/lib.rs | 1 - types/src/v0/impls/instance_state.rs | 7 ------- 2 files changed, 8 deletions(-) diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index e6eab1ca28..1d13e38029 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -979,7 +979,6 @@ pub mod testing { ) .with_current_version(V::Base::version()) .with_genesis(state) - .with_epoch_height(config.epoch_height) .with_upgrades(upgrades); // Create the HotShot membership diff --git a/types/src/v0/impls/instance_state.rs b/types/src/v0/impls/instance_state.rs index 545b3722ce..99d4e4d10b 100644 --- a/types/src/v0/impls/instance_state.rs +++ b/types/src/v0/impls/instance_state.rs @@ -135,13 +135,6 @@ impl NodeState { self.current_version = ver; self } - - // TODO remove following `Memberships` trait update: - // https://github.com/EspressoSystems/HotShot/issues/3966 - pub fn with_epoch_height(mut self, epoch_height: u64) -> Self { - self.epoch_height = Some(epoch_height); - self - } } // This allows us to turn on `Default` on InstanceState trait From a980b6c2d27d11737378e9ec0dec9aca8d58691e Mon Sep 17 00:00:00 2001 From: tbro Date: Thu, 13 Feb 2025 15:13:24 -0300 Subject: [PATCH 36/77] add a mock_v3 to `InstanceState` --- types/src/v0/impls/instance_state.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/types/src/v0/impls/instance_state.rs b/types/src/v0/impls/instance_state.rs index 99d4e4d10b..115c8050aa 100644 --- a/types/src/v0/impls/instance_state.rs +++ b/types/src/v0/impls/instance_state.rs @@ -97,6 +97,20 @@ impl NodeState { ) } + #[cfg(any(test, feature = "testing"))] + pub fn mock_v3() -> Self { + use vbs::version::StaticVersion; + + Self::new( + 0, + ChainConfig::default(), + L1Client::new(vec!["http://localhost:3331".parse().unwrap()]) + .expect("Failed to create L1 client"), + mock::MockStateCatchup::default(), + StaticVersion::<0, 3>::version(), + ) + } + #[cfg(any(test, feature = "testing"))] pub fn mock_v99() -> Self { use vbs::version::StaticVersion; From dfb0a88aa681f18f37d3787292374a95b02a8dd6 Mon Sep 17 00:00:00 2001 From: tbro Date: Fri, 14 Feb 2025 11:04:32 -0300 Subject: [PATCH 37/77] log error on ChainConfig upgrade in Header::new --- types/src/v0/impls/header.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/types/src/v0/impls/header.rs b/types/src/v0/impls/header.rs index 427dd2bf21..e0164cf653 100644 --- a/types/src/v0/impls/header.rs +++ b/types/src/v0/impls/header.rs @@ -979,7 +979,10 @@ impl BlockHeader for Header { Some(upgrade) => match upgrade.upgrade_type { UpgradeType::Fee { chain_config } => chain_config, UpgradeType::Epoch { chain_config } => chain_config, - _ => Header::get_chain_config(&validated_state, instance_state).await?, + _ => { + tracing::error!("Header::new() ChainConfig Upgrade: Unknown UpgradeType"); + Header::get_chain_config(&validated_state, instance_state).await?; + } }, None => Header::get_chain_config(&validated_state, instance_state).await?, } From 88b8280f245346078e4399b1b2d2666fde4a92ef Mon Sep 17 00:00:00 2001 From: tbro Date: Fri, 14 Feb 2025 11:10:43 -0300 Subject: [PATCH 38/77] fix --- types/src/v0/impls/header.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/src/v0/impls/header.rs b/types/src/v0/impls/header.rs index e0164cf653..c1a92f46a2 100644 --- a/types/src/v0/impls/header.rs +++ b/types/src/v0/impls/header.rs @@ -981,7 +981,7 @@ impl BlockHeader for Header { UpgradeType::Epoch { chain_config } => chain_config, _ => { tracing::error!("Header::new() ChainConfig Upgrade: Unknown UpgradeType"); - Header::get_chain_config(&validated_state, instance_state).await?; + Header::get_chain_config(&validated_state, instance_state).await? } }, None => Header::get_chain_config(&validated_state, instance_state).await?, From f61e7994befb955b5511838dffff23f3cf6c91ac Mon Sep 17 00:00:00 2001 From: tbro Date: Fri, 14 Feb 2025 12:47:58 -0300 Subject: [PATCH 39/77] Update sqlx (#2617) Updating sqlx to `0.8.3` appears to resolve. Excessive memory usage when compiling bundled sqlite. It was originally planned to add `sqlite-unbundled` as well to use native/system sqlite, but apparently excessive memory usage disappears w/ out that change. And it may be disruptive as our deployments currently rely on the bundled sqlite. As a followup, we can consider adding our own `sqlite-unbundled` feature to easily avoid compiling sqlite where we know there is one available in testing. --------- Co-authored-by: tbro --- Cargo.lock | 71 +++++++++++++------------------------ Cargo.toml | 2 +- sequencer-sqlite/Cargo.lock | 67 ++++++++++++---------------------- 3 files changed, 47 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5566a180f4..6722fe2698 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1806,7 +1806,7 @@ dependencies = [ "bitflags 2.6.0", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.13.0", "log", "prettyplease", "proc-macro2", @@ -4649,11 +4649,11 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.2", ] [[package]] @@ -6593,7 +6593,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -10400,21 +10400,11 @@ dependencies = [ "der", ] -[[package]] -name = "sqlformat" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bba3a93db0cc4f7bdece8bb09e77e2e785c20bfebf79eb8340ed80708048790" -dependencies = [ - "nom", - "unicode_categories", -] - [[package]] name = "sqlx" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93334716a037193fac19df402f8571269c84a00852f6a7066b5d2616dcd64d3e" +checksum = "4410e73b3c0d8442c5f99b425d7a435b5ee0ae4167b3196771dd3f7a01be745f" dependencies = [ "sqlx-core", "sqlx-macros", @@ -10425,39 +10415,33 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d8060b456358185f7d50c55d9b5066ad956956fddec42ee2e8567134a8936e" +checksum = "6a007b6936676aa9ab40207cde35daab0a04b823be8ae004368c0793b96a61e0" dependencies = [ - "atoi", "bit-vec 0.6.3", - "byteorder", "bytes 1.9.0", "crc", "crossbeam-queue", "either", "event-listener 5.4.0", - "futures-channel", "futures-core", "futures-intrusive", "futures-io", "futures-util", - "hashbrown 0.14.5", - "hashlink 0.9.1", - "hex", + "hashbrown 0.15.2", + "hashlink 0.10.0", "indexmap 2.7.0", "log", "memchr", "native-tls", "once_cell", - "paste", "percent-encoding", "serde", "serde_json", "sha2 0.10.8", "smallvec", - "sqlformat", - "thiserror 1.0.69", + "thiserror 2.0.10", "time 0.3.37", "tokio", "tokio-stream", @@ -10467,9 +10451,9 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac0692bcc9de3b073e8d747391827297e075c7710ff6276d9f7a1f3d58c6657" +checksum = "3112e2ad78643fef903618d78cf0aec1cb3134b019730edb039b69eaf531f310" dependencies = [ "proc-macro2", "quote", @@ -10480,9 +10464,9 @@ dependencies = [ [[package]] name = "sqlx-macros-core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1804e8a7c7865599c9c79be146dc8a9fd8cc86935fa641d3ea58e5f0688abaa5" +checksum = "4e9f90acc5ab146a99bf5061a7eb4976b573f560bc898ef3bf8435448dd5e7ad" dependencies = [ "dotenvy", "either", @@ -10506,9 +10490,9 @@ dependencies = [ [[package]] name = "sqlx-mysql" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64bb4714269afa44aef2755150a0fc19d756fb580a67db8885608cf02f47d06a" +checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233" dependencies = [ "atoi", "base64 0.22.1", @@ -10541,7 +10525,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 1.0.69", + "thiserror 2.0.10", "time 0.3.37", "tracing", "whoami", @@ -10549,9 +10533,9 @@ dependencies = [ [[package]] name = "sqlx-postgres" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa91a732d854c5d7726349bb4bb879bb9478993ceb764247660aee25f67c2f8" +checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613" dependencies = [ "atoi", "base64 0.22.1", @@ -10563,7 +10547,6 @@ dependencies = [ "etcetera", "futures-channel", "futures-core", - "futures-io", "futures-util", "hex", "hkdf 0.12.4", @@ -10581,7 +10564,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 1.0.69", + "thiserror 2.0.10", "time 0.3.37", "tracing", "whoami", @@ -10589,9 +10572,9 @@ dependencies = [ [[package]] name = "sqlx-sqlite" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5b2cf34a45953bfd3daaf3db0f7a7878ab9b7a6b91b422d24a7a9e4c857b680" +checksum = "f85ca71d3a5b24e64e1d08dd8fe36c6c95c339a896cc33068148906784620540" dependencies = [ "atoi", "flume 0.11.1", @@ -11956,12 +11939,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - [[package]] name = "universal-hash" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 94308888ad..7c7d9278b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -194,7 +194,7 @@ log-panics = { version = "2.0", features = ["with-backtrace"] } lru = "0.12" strum = { version = "0.26", features = ["derive"] } surf-disco = "0.9" -sqlx = "=0.8.2" +sqlx = "=0.8.3" tagged-base64 = "0.4" tide-disco = "0.9.3" thiserror = "1.0.69" diff --git a/sequencer-sqlite/Cargo.lock b/sequencer-sqlite/Cargo.lock index a2d52a31c8..262dcb4025 100644 --- a/sequencer-sqlite/Cargo.lock +++ b/sequencer-sqlite/Cargo.lock @@ -4437,11 +4437,11 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.2", ] [[package]] @@ -9793,21 +9793,11 @@ dependencies = [ "der", ] -[[package]] -name = "sqlformat" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bba3a93db0cc4f7bdece8bb09e77e2e785c20bfebf79eb8340ed80708048790" -dependencies = [ - "nom", - "unicode_categories", -] - [[package]] name = "sqlx" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93334716a037193fac19df402f8571269c84a00852f6a7066b5d2616dcd64d3e" +checksum = "4410e73b3c0d8442c5f99b425d7a435b5ee0ae4167b3196771dd3f7a01be745f" dependencies = [ "sqlx-core", "sqlx-macros", @@ -9818,39 +9808,33 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d8060b456358185f7d50c55d9b5066ad956956fddec42ee2e8567134a8936e" +checksum = "6a007b6936676aa9ab40207cde35daab0a04b823be8ae004368c0793b96a61e0" dependencies = [ - "atoi", "bit-vec", - "byteorder", "bytes 1.9.0", "crc", "crossbeam-queue", "either", "event-listener 5.4.0", - "futures-channel", "futures-core", "futures-intrusive", "futures-io", "futures-util", - "hashbrown 0.14.5", - "hashlink 0.9.1", - "hex", + "hashbrown 0.15.2", + "hashlink 0.10.0", "indexmap 2.7.0", "log", "memchr", "native-tls", "once_cell", - "paste", "percent-encoding", "serde", "serde_json", "sha2 0.10.8", "smallvec", - "sqlformat", - "thiserror 1.0.69", + "thiserror 2.0.10", "time 0.3.37", "tokio", "tokio-stream", @@ -9860,9 +9844,9 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac0692bcc9de3b073e8d747391827297e075c7710ff6276d9f7a1f3d58c6657" +checksum = "3112e2ad78643fef903618d78cf0aec1cb3134b019730edb039b69eaf531f310" dependencies = [ "proc-macro2", "quote", @@ -9873,9 +9857,9 @@ dependencies = [ [[package]] name = "sqlx-macros-core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1804e8a7c7865599c9c79be146dc8a9fd8cc86935fa641d3ea58e5f0688abaa5" +checksum = "4e9f90acc5ab146a99bf5061a7eb4976b573f560bc898ef3bf8435448dd5e7ad" dependencies = [ "dotenvy", "either", @@ -9899,9 +9883,9 @@ dependencies = [ [[package]] name = "sqlx-mysql" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64bb4714269afa44aef2755150a0fc19d756fb580a67db8885608cf02f47d06a" +checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233" dependencies = [ "atoi", "base64 0.22.1", @@ -9934,7 +9918,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 1.0.69", + "thiserror 2.0.10", "time 0.3.37", "tracing", "whoami", @@ -9942,9 +9926,9 @@ dependencies = [ [[package]] name = "sqlx-postgres" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa91a732d854c5d7726349bb4bb879bb9478993ceb764247660aee25f67c2f8" +checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613" dependencies = [ "atoi", "base64 0.22.1", @@ -9956,7 +9940,6 @@ dependencies = [ "etcetera", "futures-channel", "futures-core", - "futures-io", "futures-util", "hex", "hkdf 0.12.4", @@ -9974,7 +9957,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 1.0.69", + "thiserror 2.0.10", "time 0.3.37", "tracing", "whoami", @@ -9982,9 +9965,9 @@ dependencies = [ [[package]] name = "sqlx-sqlite" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5b2cf34a45953bfd3daaf3db0f7a7878ab9b7a6b91b422d24a7a9e4c857b680" +checksum = "f85ca71d3a5b24e64e1d08dd8fe36c6c95c339a896cc33068148906784620540" dependencies = [ "atoi", "flume 0.11.1", @@ -11275,12 +11258,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - [[package]] name = "universal-hash" version = "0.4.0" From a94027ad9a4fe47a507a7d9dbd3a7d0b50fed798 Mon Sep 17 00:00:00 2001 From: tbro Date: Fri, 14 Feb 2025 14:14:37 -0300 Subject: [PATCH 40/77] Cleanup chain_config upgrade in `Header::new_legacy` And add some tests --- types/src/v0/impls/header.rs | 14 +--- types/src/v0/impls/instance_state.rs | 96 +++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 16 deletions(-) diff --git a/types/src/v0/impls/header.rs b/types/src/v0/impls/header.rs index c1a92f46a2..bb7b955ae2 100644 --- a/types/src/v0/impls/header.rs +++ b/types/src/v0/impls/header.rs @@ -974,19 +974,7 @@ impl BlockHeader for Header { let mut validated_state = parent_state.clone(); - let chain_config = if version > instance_state.current_version { - match instance_state.upgrades.get(&version) { - Some(upgrade) => match upgrade.upgrade_type { - UpgradeType::Fee { chain_config } => chain_config, - UpgradeType::Epoch { chain_config } => chain_config, - _ => { - tracing::error!("Header::new() ChainConfig Upgrade: Unknown UpgradeType"); - Header::get_chain_config(&validated_state, instance_state).await? - } - }, - None => Header::get_chain_config(&validated_state, instance_state).await?, - } - } else { + let chain_config = instance_state.upgrade_chain_config(version) else { Header::get_chain_config(&validated_state, instance_state).await? }; diff --git a/types/src/v0/impls/instance_state.rs b/types/src/v0/impls/instance_state.rs index 115c8050aa..b7f97484ec 100644 --- a/types/src/v0/impls/instance_state.rs +++ b/types/src/v0/impls/instance_state.rs @@ -9,7 +9,7 @@ use vbs::version::Version; #[cfg(any(test, feature = "testing"))] use vbs::version::{StaticVersion, StaticVersionType}; -use super::state::ValidatedState; +use super::{state::ValidatedState, UpgradeType}; /// Represents the immutable state of a node. /// @@ -145,10 +145,24 @@ impl NodeState { self } - pub fn with_current_version(mut self, ver: Version) -> Self { - self.current_version = ver; + pub fn with_current_version(mut self, version: Version) -> Self { + self.current_version = version; self } + + /// Given a `version`, get the correct `ChainConfig` from `self.upgrades`. + pub fn upgrade_chain_config(&self, version: Version) -> Option { + let chain_config = (version > self.current_version).then(|| { + self.upgrades + .get(&version) + .and_then(|upgrade| match upgrade.upgrade_type { + UpgradeType::Fee { chain_config } => Some(chain_config), + UpgradeType::Epoch { chain_config } => Some(chain_config), + _ => None, + }) + }); + chain_config? + } } // This allows us to turn on `Default` on InstanceState trait @@ -289,3 +303,79 @@ pub mod mock { } } } + +#[cfg(test)] +mod test { + + use crate::v0::Versions; + use crate::{EpochVersion, FeeVersion, SequencerVersions, ViewBasedUpgrade}; + + use super::*; + + #[test] + fn test_upgrade_chain_config_version_02() { + let mut upgrades = std::collections::BTreeMap::new(); + type MySequencerVersions = SequencerVersions, FeeVersion>; + + let mode = UpgradeMode::View(ViewBasedUpgrade { + start_voting_view: None, + stop_voting_view: None, + start_proposing_view: 1, + stop_proposing_view: 10, + }); + + let upgraded_chain_config = ChainConfig { + max_block_size: 300.into(), + base_fee: 1.into(), + ..Default::default() + }; + + let upgrade_type = UpgradeType::Fee { + chain_config: upgraded_chain_config, + }; + + upgrades.insert( + ::Upgrade::VERSION, + Upgrade { mode, upgrade_type }, + ); + + let instance_state = NodeState::mock().with_upgrades(upgrades); + + let chain_config = instance_state.upgrade_chain_config(FeeVersion::version()); + assert_eq!(Some(upgraded_chain_config), chain_config); + } + + #[test] + fn test_upgrade_chain_config_version_03() { + let mut upgrades = std::collections::BTreeMap::new(); + type MySequencerVersions = SequencerVersions; + + let mode = UpgradeMode::View(ViewBasedUpgrade { + start_voting_view: None, + stop_voting_view: None, + start_proposing_view: 1, + stop_proposing_view: 10, + }); + + let upgraded_chain_config = ChainConfig { + max_block_size: 300.into(), + base_fee: 1.into(), + stake_table_contract: Some(Default::default()), + ..Default::default() + }; + + let upgrade_type = UpgradeType::Epoch { + chain_config: upgraded_chain_config, + }; + + upgrades.insert( + ::Upgrade::VERSION, + Upgrade { mode, upgrade_type }, + ); + + let instance_state = NodeState::mock_v2().with_upgrades(upgrades); + + let chain_config = instance_state.upgrade_chain_config(EpochVersion::version()); + assert_eq!(Some(upgraded_chain_config), chain_config); + } +} From 8eaef68fd9724ffff374a5aaca30386cf629af43 Mon Sep 17 00:00:00 2001 From: Abdul Basit <45506001+imabdulbasit@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:26:01 +0500 Subject: [PATCH 41/77] add initial stakers through stake table update txn AND no multi sig ownership transfer (#2580) add stakers through update txn && do not transfer ownership --- .env | 4 +-- data/genesis/demo-pos-base.toml | 2 +- data/genesis/demo-pos.toml | 2 +- data/initial_stake_table.toml | 10 ++++---- process-compose.yaml | 19 ++++++++++++-- sequencer/src/bin/deploy.rs | 25 ------------------- .../bin/update-permissioned-stake-table.rs | 6 ++++- sequencer/src/lib.rs | 2 +- types/src/v0/impls/stake_table.rs | 8 ++++-- utils/src/deployer.rs | 4 +-- utils/src/stake_table.rs | 5 ++++ 11 files changed, 44 insertions(+), 43 deletions(-) diff --git a/.env b/.env index 8688fbabdc..8121b24844 100644 --- a/.env +++ b/.env @@ -57,10 +57,10 @@ ESPRESSO_BUILDER_ETH_ACCOUNT_INDEX=8 ESPRESSO_DEPLOYER_ACCOUNT_INDEX=9 # Contracts -ESPRESSO_SEQUENCER_LIGHT_CLIENT_PROXY_ADDRESS=0x0c8e79f3534b00d9a3d4a856b665bf4ebc22f2ba +ESPRESSO_SEQUENCER_LIGHT_CLIENT_PROXY_ADDRESS=0xe1da8919f262ee86f9be05059c9280142cf23f48 ESPRESSO_SEQUENCER_LIGHTCLIENT_ADDRESS=$ESPRESSO_SEQUENCER_LIGHT_CLIENT_PROXY_ADDRESS ESPRESSO_SEQUENCER_PERMISSIONED_PROVER=0x14dc79964da2c08b23698b3d3cc7ca32193d9955 -ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS=0x8ce361602b935680e8dec218b820ff5056beb7af +ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS=0xb19b36b1456e65e3a6d514d3f715f204bd59f431 # Example sequencer demo private keys ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_0=BLS_SIGNING_KEY~lNDh4Pn-pTAyzyprOAFdXHwhrKhEwqwtMtkD3CZF4x3o diff --git a/data/genesis/demo-pos-base.toml b/data/genesis/demo-pos-base.toml index b7a5994ced..8d174799a1 100644 --- a/data/genesis/demo-pos-base.toml +++ b/data/genesis/demo-pos-base.toml @@ -11,7 +11,7 @@ base_fee = '1 wei' fee_recipient = "0x0000000000000000000000000000000000000000" # bid_recipient = "0x0000000000000000000000000000000000000000" fee_contract = "0xa15bb66138824a1c7167f5e85b957d04dd34e468" -stake_table_contract = "0x8ce361602b935680e8dec218b820ff5056beb7af" +stake_table_contract = "0xb19b36b1456e65e3a6d514d3f715f204bd59f431" [header] timestamp = "1970-01-01T00:00:00Z" diff --git a/data/genesis/demo-pos.toml b/data/genesis/demo-pos.toml index 09f60756ab..b18e972383 100644 --- a/data/genesis/demo-pos.toml +++ b/data/genesis/demo-pos.toml @@ -30,4 +30,4 @@ base_fee = '1 wei' fee_recipient = "0x0000000000000000000000000000000000000000" bid_recipient = "0x0000000000000000000000000000000000000000" fee_contract = "0xa15bb66138824a1c7167f5e85b957d04dd34e468" -stake_table_contract = "0x8ce361602b935680e8dec218b820ff5056beb7af" \ No newline at end of file +stake_table_contract = "0xb19b36b1456e65e3a6d514d3f715f204bd59f431" \ No newline at end of file diff --git a/data/initial_stake_table.toml b/data/initial_stake_table.toml index 59384a6509..57bbdffc68 100644 --- a/data/initial_stake_table.toml +++ b/data/initial_stake_table.toml @@ -1,29 +1,29 @@ -[[public_keys]] +[[new_stakers]] stake_table_key = "BLS_VER_KEY~bQszS-QKYvUij2g20VqS8asttGSb95NrTu2PUj0uMh1CBUxNy1FqyPDjZqB29M7ZbjWqj79QkEOWkpga84AmDYUeTuWmy-0P1AdKHD3ehc-dKvei78BDj5USwXPJiDUlCxvYs_9rWYhagaq-5_LXENr78xel17spftNd5MA1Mw5U" state_ver_key = "SCHNORR_VER_KEY~ibJCbfPOhDoURqiGLe683TDJ_KOLQCx8_Hdq43dOviSuL6WJJ_2mARKO3xA2k5zpXE3iiq4_z7mzvA-V1VXvIWw" da = true stake = 1 -[[public_keys]] +[[new_stakers]] stake_table_key = "BLS_VER_KEY~4zQnaCOFJ7m95OjxeNls0QOOwWbz4rfxaL3NwmN2zSdnf8t5Nw_dfmMHq05ee8jCegw6Bn5T8inmrnGGAsQJMMWLv77nd7FJziz2ViAbXg-XGGF7o4HyzELCmypDOIYF3X2UWferFE_n72ZX0iQkUhOvYZZ7cfXToXxRTtb_mwRR" state_ver_key = "SCHNORR_VER_KEY~lNCMqH5qLthH5OXxW_Z25tLXJUqmzzhsuQ6oVuaPWhtRPmgIKSqcBoJTaEbmGZL2VfTyQNguaoQL4U_4tCA_HmI" da = true stake = 1 -[[public_keys]] +[[new_stakers]] stake_table_key = "BLS_VER_KEY~IBRoz_Q1EXvcm1pNZcmVlyYZU8hZ7qmy337ePAjEMhz8Hl2q8vWPFOd3BaLwgRS1UzAPW3z4E-XIgRDGcRBTAMZX9b_0lKYjlyTlNF2EZfNnKmvv-xJ0yurkfjiveeYEsD2l5d8q_rJJbH1iZdXy-yPEbwI0SIvQfwdlcaKw9po4" state_ver_key = "SCHNORR_VER_KEY~nkFKzpLhJAafJ3LBkY_0h9OzxSyTu95Z029EUFPO4QNkeUo6DHQGTTVjxmprTA5H8jRSn73i0slJvig6dZ5kLX4" da = true stake = 1 -[[public_keys]] +[[new_stakers]] stake_table_key = "BLS_VER_KEY~rO2PIjyY30HGfapFcloFe3mNDKMIFi6JlOLkH5ZWBSYoRm5fE2-Rm6Lp3EvmAcB5r7KFJ0c1Uor308x78r04EY_sfjcsDCWt7RSJdL4cJoD_4fSTCv_bisO8k98hs_8BtqQt8BHlPeJohpUXvcfnK8suXJETiJ6Er97pfxRbzgAL" state_ver_key = "SCHNORR_VER_KEY~NwYhzlWarlZHxTNvChWuf74O3fP7zIt5NdC7V8gV6w2W92JOBDkrNmKQeMGxMUke-G5HHxUjHlZEWr1m1xLjEaI" da = false stake = 1 -[[public_keys]] +[[new_stakers]] stake_table_key = "BLS_VER_KEY~r6b-Cwzp-b3czlt0MHmYPJIow5kMsXbrNmZsLSYg9RV49oCCO4WEeCRFR02x9bqLCa_sgNFMrIeNdEa11qNiBAohApYFIvrSa-zP5QGj3xbZaMOCrshxYit6E2TR-XsWvv6gjOrypmugjyTAth-iqQzTboSfmO9DD1-gjJIdCaD7" state_ver_key = "SCHNORR_VER_KEY~qMfMj1c1hRVTnugvz3MKNnVC5JA9jvZcV3ZCLL_J4Ap-u0i6ulGWveTk3OOelZj2-kd_WD5ojtYGWV1jHx9wCaA" da = true diff --git a/process-compose.yaml b/process-compose.yaml index 14906a67a8..6699a4fe90 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -28,7 +28,9 @@ processes: command: unset ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS ESPRESSO_SEQUENCER_FEE_CONTRACT_PROXY_ADDRESS - && deploy --only fee-contract,permissioned-stake-table + && deploy --only fee-contract + && unset ESPRESSO_SEQUENCER_ETH_MULTISIG_ADDRESS + && deploy --only permissioned-stake-table namespace: setup depends_on: demo-l1-network: @@ -111,6 +113,19 @@ processes: condition: process_healthy deploy-prover-contracts: condition: process_completed + + update-permissioned-stake-table: + command: update-permissioned-stake-table + environment: + - ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS + - ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_UPDATE_TOML_PATH=data/initial_stake_table.toml + depends_on: + deploy-prover-contracts: + condition: process_completed + sequencer0: + condition: process_healthy + + sequencer0: command: sequencer -- storage-sql -- http -- query -- submit -- hotshot-events -- config @@ -534,7 +549,7 @@ processes: host: localhost port: $ESPRESSO_BUILDER_SERVER_PORT path: /healthcheck - failure_threshold: 5 + failure_threshold: 100 period_seconds: 1 availability: restart: "exit_on_failure" diff --git a/sequencer/src/bin/deploy.rs b/sequencer/src/bin/deploy.rs index 8a5916bd43..d766827cc0 100644 --- a/sequencer/src/bin/deploy.rs +++ b/sequencer/src/bin/deploy.rs @@ -9,7 +9,6 @@ use hotshot_state_prover::service::light_client_genesis; use sequencer_utils::{ deployer::{deploy, ContractGroup, Contracts, DeployedContracts}, logging, - stake_table::PermissionedStakeTableConfig, }; use url::Url; @@ -122,22 +121,6 @@ struct Options { /// If the light client contract is not being deployed, this option is ignored. #[clap(long, env = "ESPRESSO_SEQUENCER_PERMISSIONED_PROVER")] permissioned_prover: Option
, - - /// A toml file with the initial stake table. - /// - /// Schema: - /// - /// public_keys = [ - /// { - /// stake_table_key = "BLS_VER_KEY~...", - /// state_ver_key = "SCHNORR_VER_KEY~...", - /// da = true, - /// stake = 1, # this value is ignored, but needs to be set - /// }, - /// ] - #[clap(long, env = "ESPRESSO_SEQUENCER_INITIAL_PERMISSIONED_STAKE_TABLE_PATH")] - initial_stake_table_path: Option, - #[clap(flatten)] logging: logging::Config, } @@ -153,13 +136,6 @@ async fn main() -> anyhow::Result<()> { let genesis = light_client_genesis(&sequencer_url, opt.stake_table_capacity).boxed(); - let initial_stake_table = if let Some(path) = opt.initial_stake_table_path { - tracing::info!("Loading initial stake table from {:?}", path); - Some(PermissionedStakeTableConfig::from_toml_file(&path)?.into()) - } else { - None - }; - let contracts = deploy( opt.rpc_url, opt.l1_polling_interval, @@ -171,7 +147,6 @@ async fn main() -> anyhow::Result<()> { genesis, opt.permissioned_prover, contracts, - initial_stake_table, ) .await?; diff --git a/sequencer/src/bin/update-permissioned-stake-table.rs b/sequencer/src/bin/update-permissioned-stake-table.rs index 28e30767d6..890080ef3e 100644 --- a/sequencer/src/bin/update-permissioned-stake-table.rs +++ b/sequencer/src/bin/update-permissioned-stake-table.rs @@ -87,7 +87,11 @@ struct Options { async fn main() -> Result<()> { let opts = Options::parse(); opts.logging.init(); - let update = PermissionedStakeTableUpdate::from_toml_file(&opts.update_toml_path)?; + + let path = opts.update_toml_path; + + tracing::error!("updating stake table from path: {path:?}"); + let update = PermissionedStakeTableUpdate::from_toml_file(&path)?; update_stake_table( opts.rpc_url, diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 1d13e38029..08dc912fa7 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -363,7 +363,7 @@ pub async fn init_node( } //todo(abdul): get from genesis file - network_config.config.epoch_height = 150; + network_config.config.epoch_height = 10; // If the `Libp2p` bootstrap nodes were supplied via the command line, override those // present in the config file. diff --git a/types/src/v0/impls/stake_table.rs b/types/src/v0/impls/stake_table.rs index d52637cc48..f54925e8b7 100644 --- a/types/src/v0/impls/stake_table.rs +++ b/types/src/v0/impls/stake_table.rs @@ -84,6 +84,7 @@ impl StakeTables { da_members.push(node.into()); } } + Self::new(consensus_stake_table.into(), da_members.into()) } } @@ -247,12 +248,14 @@ impl EpochCommittees { map.insert(Epoch::new(epoch), members.clone()); } + let address = instance_state.chain_config.stake_table_contract; + Self { non_epoch_committee: members, state: map, _epoch_size: epoch_size, l1_client: instance_state.l1_client.clone(), - contract_address: instance_state.chain_config.stake_table_contract, + contract_address: address, } } @@ -488,8 +491,9 @@ impl Membership for EpochCommittees { block_header: Header, ) -> Option> { let address = self.contract_address?; + self.l1_client - .get_stake_table(address.to_alloy(), block_header.height()) + .get_stake_table(address.to_alloy(), block_header.l1_head()) .await .ok() .map(|stake_table| -> Box { diff --git a/utils/src/deployer.rs b/utils/src/deployer.rs index b19f249d6b..9210ceb436 100644 --- a/utils/src/deployer.rs +++ b/utils/src/deployer.rs @@ -321,7 +321,6 @@ pub async fn deploy( genesis: BoxFuture<'_, anyhow::Result<(ParsedLightClientState, ParsedStakeTableState)>>, permissioned_prover: Option
, mut contracts: Contracts, - initial_stake_table: Option>, ) -> anyhow::Result { let provider = Provider::::try_from(l1url.to_string())?.interval(l1_interval); let chain_id = provider.get_chainid().await?.as_u64(); @@ -461,11 +460,10 @@ pub async fn deploy( // `PermissionedStakeTable.sol` if should_deploy(ContractGroup::PermissionedStakeTable, &only) { - let initial_stake_table: Vec<_> = initial_stake_table.unwrap_or_default(); let stake_table_address = contracts .deploy_tx( Contract::PermissonedStakeTable, - PermissionedStakeTable::deploy(l1.clone(), initial_stake_table)?, + PermissionedStakeTable::deploy(l1.clone(), Vec::::new())?, ) .await?; let stake_table = PermissionedStakeTable::new(stake_table_address, l1.clone()); diff --git a/utils/src/stake_table.rs b/utils/src/stake_table.rs index 2dd0fd0ef5..7d01517e3e 100644 --- a/utils/src/stake_table.rs +++ b/utils/src/stake_table.rs @@ -137,12 +137,17 @@ pub async fn update_stake_table( .index(account_index)? .build()? .with_chain_id(chain_id); + let l1 = Arc::new(SignerMiddleware::new(provider.clone(), wallet)); let contract = PermissionedStakeTable::new(contract_address, l1); tracing::info!("sending stake table update transaction"); + if update.stakers_to_remove().is_empty() && update.new_stakers().is_empty() { + anyhow::bail!("No changes to update in the stake table"); + } + let tx_receipt = contract .update(update.stakers_to_remove(), update.new_stakers()) .send() From 35ad5b84fab3df0fde192534665610cb93fb9fe3 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Mon, 17 Feb 2025 14:38:16 +0500 Subject: [PATCH 42/77] fix build and clippy --- sequencer/src/run.rs | 17 +++++++---------- types/src/v0/impls/header.rs | 7 ++++--- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/sequencer/src/run.rs b/sequencer/src/run.rs index 6efe90c484..53aa25291a 100644 --- a/sequencer/src/run.rs +++ b/sequencer/src/run.rs @@ -8,10 +8,7 @@ use super::{ persistence, Genesis, L1Params, NetworkParams, }; use clap::Parser; -use espresso_types::{ - traits::NullEventConsumer, EpochVersion, FeeVersion, MarketplaceVersion, SequencerVersions, - SolverAuctionResultsProvider, V0_0, -}; +use espresso_types::{traits::NullEventConsumer, SequencerVersions, SolverAuctionResultsProvider}; use futures::future::FutureExt; use hotshot::MarketplaceConfig; use hotshot_types::traits::{metrics::NoMetrics, node_implementation::Versions}; @@ -39,34 +36,34 @@ pub async fn main() -> anyhow::Result<()> { match (base, upgrade) { #[cfg(all(feature = "fee", feature = "pos"))] - (FeeVersion::VERSION, EpochVersion::VERSION) => { + (espresso_types::FeeVersion::VERSION, espresso_types::EpochVersion::VERSION) => { run( genesis, modules, opt, - SequencerVersions::::new(), + SequencerVersions::::new(), ) .await } #[cfg(feature = "pos")] - (EpochVersion::VERSION, _) => { + (espresso_types::EpochVersion::VERSION, _) => { run( genesis, modules, opt, // Specifying V0_0 disables upgrades - SequencerVersions::::new(), + SequencerVersions::::new(), ) .await } // TODO change `fee` to `pos` #[cfg(all(feature = "fee", feature = "marketplace"))] - (FeeVersion::VERSION, MarketplaceVersion::VERSION) => { + (espresso_types::FeeVersion::VERSION, espresso_types::MarketplaceVersion::VERSION) => { run( genesis, modules, opt, - SequencerVersions::::new(), + SequencerVersions::::new(), ) .await } diff --git a/types/src/v0/impls/header.rs b/types/src/v0/impls/header.rs index bb7b955ae2..0c54a265dc 100644 --- a/types/src/v0/impls/header.rs +++ b/types/src/v0/impls/header.rs @@ -33,7 +33,7 @@ use crate::{ v0_1, v0_2, v0_3, v0_99::{self, ChainConfig, IterableFeeInfo, SolverAuctionResults}, BlockMerkleCommitment, BuilderSignature, FeeAccount, FeeAmount, FeeInfo, FeeMerkleCommitment, - Header, L1BlockInfo, L1Snapshot, Leaf2, NamespaceId, NsTable, SeqTypes, UpgradeType, + Header, L1BlockInfo, L1Snapshot, Leaf2, NamespaceId, NsTable, SeqTypes, }; use super::{instance_state::NodeState, state::ValidatedState}; @@ -974,8 +974,9 @@ impl BlockHeader for Header { let mut validated_state = parent_state.clone(); - let chain_config = instance_state.upgrade_chain_config(version) else { - Header::get_chain_config(&validated_state, instance_state).await? + let chain_config = match instance_state.upgrade_chain_config(version) { + Some(chain_config) => chain_config, + None => Header::get_chain_config(&validated_state, instance_state).await?, }; validated_state.chain_config = chain_config.into(); From 9fee0895ca603e78b9fbf15ab3194b56a6cbe0b6 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Mon, 17 Feb 2025 14:57:18 +0500 Subject: [PATCH 43/77] revert justfile --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 2aa0f43e95..84b4493845 100644 --- a/justfile +++ b/justfile @@ -1,4 +1,4 @@ -# export RUST_BACKTRACE := "1" +mod hotshot default: just --list From ac7d8112b6f0b5ccbe73f7c0e79e472aa6b5bed4 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Mon, 17 Feb 2025 15:45:25 +0500 Subject: [PATCH 44/77] fix devnode --- sequencer/src/bin/espresso-dev-node.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/sequencer/src/bin/espresso-dev-node.rs b/sequencer/src/bin/espresso-dev-node.rs index 7fed6cec03..c80fb4c72d 100644 --- a/sequencer/src/bin/espresso-dev-node.rs +++ b/sequencer/src/bin/espresso-dev-node.rs @@ -292,7 +292,6 @@ async fn main() -> anyhow::Result<()> { async { Ok(lc_genesis.clone()) }.boxed(), None, contracts.clone(), - None, // initial stake table ) .await?; From b45011d1851c7bd4984f3bc8d1d21d4f3cdd1072 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Mon, 17 Feb 2025 16:21:02 +0500 Subject: [PATCH 45/77] do not validate and apply header for block in epoch transition --- sequencer/src/lib.rs | 2 +- types/src/v0/impls/instance_state.rs | 4 ++-- types/src/v0/impls/state.rs | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 08dc912fa7..d57c4fdc5d 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -483,7 +483,7 @@ pub async fn init_node( node_id: node_index, upgrades: genesis.upgrades, current_version: V::Base::VERSION, - epoch_height: None, + epoch_height: network_config.config.epoch_height, }; // Create the HotShot membership diff --git a/types/src/v0/impls/instance_state.rs b/types/src/v0/impls/instance_state.rs index b7f97484ec..afa4d681ca 100644 --- a/types/src/v0/impls/instance_state.rs +++ b/types/src/v0/impls/instance_state.rs @@ -24,7 +24,7 @@ pub struct NodeState { pub genesis_header: GenesisHeader, pub genesis_state: ValidatedState, pub l1_genesis: Option, - pub epoch_height: Option, + pub epoch_height: u64, /// Map containing all planned and executed upgrades. /// @@ -65,7 +65,7 @@ impl NodeState { l1_genesis: None, upgrades: Default::default(), current_version, - epoch_height: None, + epoch_height: 0, } } diff --git a/types/src/v0/impls/state.rs b/types/src/v0/impls/state.rs index a5ab7cfec0..453f33a796 100644 --- a/types/src/v0/impls/state.rs +++ b/types/src/v0/impls/state.rs @@ -909,6 +909,12 @@ impl HotShotState for ValidatedState { version: Version, view_number: u64, ) -> Result<(Self, Self::Delta), Self::Error> { + if proposed_header.height() % instance.epoch_height == 0 + && parent_leaf.height() == proposed_header.height() + { + return Ok((self.clone(), Delta::default())); + } + // Unwrapping here is okay as we retry in a loop //so we should either get a validated state or until hotshot cancels the task let (validated_state, delta) = self From 103b5e2e607a736d7b121f42b89abd68194219bb Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Mon, 17 Feb 2025 16:24:15 +0500 Subject: [PATCH 46/77] add comments to validate_and_apply_header() --- types/src/v0/impls/state.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/types/src/v0/impls/state.rs b/types/src/v0/impls/state.rs index 453f33a796..aecc11c7a6 100644 --- a/types/src/v0/impls/state.rs +++ b/types/src/v0/impls/state.rs @@ -909,6 +909,9 @@ impl HotShotState for ValidatedState { version: Version, view_number: u64, ) -> Result<(Self, Self::Delta), Self::Error> { + // During epoch transition, hotshot propagates the same block again + // we should totally skip this block, and return the same validated state + // This block will have the same parent block height if proposed_header.height() % instance.epoch_height == 0 && parent_leaf.height() == proposed_header.height() { From cd9c06d69bda04cf67c78b37559dad935ea46361 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Mon, 17 Feb 2025 17:22:59 +0500 Subject: [PATCH 47/77] fix log --- types/src/v0/impls/state.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/types/src/v0/impls/state.rs b/types/src/v0/impls/state.rs index aecc11c7a6..2efa503442 100644 --- a/types/src/v0/impls/state.rs +++ b/types/src/v0/impls/state.rs @@ -912,9 +912,23 @@ impl HotShotState for ValidatedState { // During epoch transition, hotshot propagates the same block again // we should totally skip this block, and return the same validated state // This block will have the same parent block height + + tracing::info!( + "parent_height={} proposed_height={}", + parent_leaf.height(), + proposed_header.height(), + ); + if proposed_header.height() % instance.epoch_height == 0 && parent_leaf.height() == proposed_header.height() { + tracing::info!( + "skipping block.. parent_height={} proposed_height={} epoch_height={}", + parent_leaf.height(), + proposed_header.height(), + instance.epoch_height, + ); + return Ok((self.clone(), Delta::default())); } From 35aaa4728d460eb8f9b9cf17fd07ad36013e44d2 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 18 Feb 2025 02:17:44 +0500 Subject: [PATCH 48/77] todo: l1 events order --- types/src/v0/impls/stake_table.rs | 77 ++++++++++++++++++++++++------- types/src/v0/impls/state.rs | 4 +- 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/types/src/v0/impls/stake_table.rs b/types/src/v0/impls/stake_table.rs index f54925e8b7..f7c5651b5a 100644 --- a/types/src/v0/impls/stake_table.rs +++ b/types/src/v0/impls/stake_table.rs @@ -189,6 +189,8 @@ impl EpochCommittees { }; self.state.insert(epoch, committee.clone()); + self.state.insert(epoch + 1, committee.clone()); + self.state.insert(epoch + 2, committee.clone()); committee } @@ -341,19 +343,25 @@ impl Membership for EpochCommittees { /// Get the stake table for the current view fn stake_table(&self, epoch: Option) -> Vec> { - if let Some(st) = self.state(&epoch) { + let st = if let Some(st) = self.state(&epoch) { st.stake_table.clone() } else { vec![] - } + }; + + tracing::debug!("stake table = {st:?}"); + st } /// Get the stake table for the current view fn da_stake_table(&self, epoch: Option) -> Vec> { - if let Some(sc) = self.state(&epoch) { + let da = if let Some(sc) = self.state(&epoch) { sc.da_members.clone() } else { vec![] - } + }; + + tracing::debug!("da members = {da:?}"); + da } /// Get all members of the committee for the current view @@ -362,11 +370,15 @@ impl Membership for EpochCommittees { _view_number: ::View, epoch: Option, ) -> BTreeSet { - if let Some(sc) = self.state(&epoch) { + let committee = if let Some(sc) = self.state(&epoch) { sc.indexed_stake_table.clone().into_keys().collect() } else { BTreeSet::new() - } + }; + + tracing::debug!("committee={committee:?}"); + + committee } /// Get all members of the committee for the current view @@ -375,11 +387,14 @@ impl Membership for EpochCommittees { _view_number: ::View, epoch: Option, ) -> BTreeSet { - if let Some(sc) = self.state(&epoch) { + let da = if let Some(sc) = self.state(&epoch) { sc.indexed_da_members.clone().into_keys().collect() } else { BTreeSet::new() - } + }; + tracing::debug!("da committee={da:?}"); + + da } /// Get all eligible leaders of the committee for the current view @@ -388,12 +403,16 @@ impl Membership for EpochCommittees { _view_number: ::View, epoch: Option, ) -> BTreeSet { - self.state(&epoch) + let committee_leaders = self + .state(&epoch) .unwrap() .eligible_leaders .iter() .map(PubKey::public_key) - .collect() + .collect(); + + tracing::debug!("committee_leaders={committee_leaders:?}"); + committee_leaders } /// Get the stake table entry for a public key @@ -436,6 +455,8 @@ impl Membership for EpochCommittees { .eligible_leaders .clone(); + tracing::debug!("lookup_leader() leaders={leaders:?}"); + let index = *view_number as usize % leaders.len(); let res = leaders[index].clone(); Ok(PubKey::public_key(&res)) @@ -490,17 +511,39 @@ impl Membership for EpochCommittees { epoch: Epoch, block_header: Header, ) -> Option> { + + // TODO: (abdul) fix fetching from contracts + // so that order of l1 events match with the update let address = self.contract_address?; + let genesis_st = self.state(&Some(Epoch::genesis())).unwrap().clone(); - self.l1_client + let st = self + .l1_client .get_stake_table(address.to_alloy(), block_header.l1_head()) .await - .ok() - .map(|stake_table| -> Box { - Box::new(move |committee: &mut Self| { - let _ = committee.update_stake_table(epoch, stake_table); - }) - }) + .ok(); + + let epoch_0_st = genesis_st.stake_table; + let epoch0_da = genesis_st.da_members; + + let sss = st.clone().unwrap(); + let contract_st = sss.stake_table; + let contract_da = sss.da_members; + + tracing::warn!("epoch0 st= {epoch_0_st:?}"); + tracing::warn!("contract st= {contract_st:?}"); + + tracing::warn!("epoch0 da= {contract_da:?}"); + tracing::warn!("contact da= {epoch0_da:?}"); + + let stake_tables = StakeTables { + stake_table: epoch_0_st.into(), + da_members: epoch0_da.into(), + }; + + Some(Box::new(move |committee: &mut Self| { + let _ = committee.update_stake_table(epoch, stake_tables); + })) } } diff --git a/types/src/v0/impls/state.rs b/types/src/v0/impls/state.rs index 2efa503442..9876c3ff96 100644 --- a/types/src/v0/impls/state.rs +++ b/types/src/v0/impls/state.rs @@ -913,7 +913,7 @@ impl HotShotState for ValidatedState { // we should totally skip this block, and return the same validated state // This block will have the same parent block height - tracing::info!( + tracing::error!( "parent_height={} proposed_height={}", parent_leaf.height(), proposed_header.height(), @@ -922,7 +922,7 @@ impl HotShotState for ValidatedState { if proposed_header.height() % instance.epoch_height == 0 && parent_leaf.height() == proposed_header.height() { - tracing::info!( + tracing::error!( "skipping block.. parent_height={} proposed_height={} epoch_height={}", parent_leaf.height(), proposed_header.height(), From a2a764c7bffbcd834408c8892f1104f7d173bae9 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 18 Feb 2025 16:48:19 +0500 Subject: [PATCH 49/77] save initial stake table from hotshot config --- flake.nix | 10 ++-------- sequencer/src/lib.rs | 7 +++++++ types/src/v0/impls/stake_table.rs | 1 - types/src/v0/impls/state.rs | 2 +- utils/src/stake_table.rs | 32 ++++++++++++++++++++++++++++++- 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/flake.nix b/flake.nix index 47b0a13bba..be4c42ecc4 100644 --- a/flake.nix +++ b/flake.nix @@ -67,13 +67,7 @@ solhintPkg { inherit (prev) buildNpmPackage fetchFromGitHub; }; }) - # The mold linker is around 50% faster on Linux than the default linker. - # This overlays a mkShell that is configured to use mold on Linux. - (final: prev: prev.lib.optionalAttrs prev.stdenv.isLinux { - mkShell = prev.mkShell.override { - stdenv = prev.stdenvAdapters.useMoldLinker prev.clangStdenv; - }; - }) + ]; pkgs = import nixpkgs { inherit system overlays; }; crossShell = { config }: @@ -323,4 +317,4 @@ ]; }); }); -} +} \ No newline at end of file diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index d57c4fdc5d..79eb77c133 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -23,6 +23,7 @@ use espresso_types::{ use ethers_conv::ToAlloy; use genesis::L1Finalized; use proposal_fetcher::ProposalFetcherConfig; +use sequencer_utils::stake_table::PermissionedStakeTableUpdate; use std::sync::Arc; use tokio::select; // Should move `STAKE_TABLE_CAPACITY` in the sequencer repo when we have variate stake table support @@ -494,6 +495,12 @@ pub async fn init_node( network_config.config.epoch_height, ); + // save initial stake table into toml file + // this will be helpful to load it into contract + PermissionedStakeTableUpdate::save_initial_stake_table_from_hotshot_config( + network_config.config.clone(), + )?; + // Initialize the Libp2p network let network = { let p2p_network = Libp2pNetwork::from_config( diff --git a/types/src/v0/impls/stake_table.rs b/types/src/v0/impls/stake_table.rs index f7c5651b5a..0ba7f1ddae 100644 --- a/types/src/v0/impls/stake_table.rs +++ b/types/src/v0/impls/stake_table.rs @@ -546,7 +546,6 @@ impl Membership for EpochCommittees { })) } } - #[cfg(test)] mod tests { use contract_bindings_alloy::permissionedstaketable::PermissionedStakeTable::NodeInfo; diff --git a/types/src/v0/impls/state.rs b/types/src/v0/impls/state.rs index 9876c3ff96..f192a74274 100644 --- a/types/src/v0/impls/state.rs +++ b/types/src/v0/impls/state.rs @@ -913,7 +913,7 @@ impl HotShotState for ValidatedState { // we should totally skip this block, and return the same validated state // This block will have the same parent block height - tracing::error!( + tracing::debug!( "parent_height={} proposed_height={}", parent_leaf.height(), proposed_header.height(), diff --git a/utils/src/stake_table.rs b/utils/src/stake_table.rs index 7d01517e3e..414991f654 100644 --- a/utils/src/stake_table.rs +++ b/utils/src/stake_table.rs @@ -14,7 +14,7 @@ use ethers::{ }; use hotshot::types::BLSPubKey; use hotshot_contract_adapter::stake_table::{bls_jf_to_sol, NodeInfoJf}; -use hotshot_types::network::PeerConfigKeys; +use hotshot_types::{network::PeerConfigKeys, traits::signature_key::StakeTableEntryType, HotShotConfig}; use url::Url; use std::{fs, path::Path, sync::Arc, time::Duration}; @@ -104,6 +104,16 @@ impl PermissionedStakeTableUpdate { ) } + pub fn to_toml_file(&self, path: &Path) -> anyhow::Result<()> { + let toml_string = toml::to_string_pretty(self) + .unwrap_or_else(|err| panic!("Failed to serialize config to TOML: {err}")); + + fs::write(path, toml_string) + .unwrap_or_else(|_| panic!("Could not write config file to {}", path.display())); + + Ok(()) + } + fn stakers_to_remove(&self) -> Vec { self.stakers_to_remove .iter() @@ -120,6 +130,26 @@ impl PermissionedStakeTableUpdate { }) .collect() } + + pub fn save_initial_stake_table_from_hotshot_config( + config: HotShotConfig, + ) -> anyhow::Result<()> { + let committee_members = config.known_nodes_with_stake.clone(); + let known_da_nodes = config.known_da_nodes.clone().clone(); + let members = committee_members + .into_iter() + .map(|m| PeerConfigKeys { + stake_table_key: m.stake_table_entry.public_key(), + state_ver_key: m.state_ver_key.clone(), + stake: m.stake_table_entry.stake().as_u64(), + da: known_da_nodes.contains(&m), + }) + .collect(); + + Self::new(members, vec![]).to_toml_file(Path::new("data/initial_stake_table.toml"))?; + + Ok(()) + } } pub async fn update_stake_table( From bc7dee590a3ae69d26c932ddca04c0c2745bb272 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 18 Feb 2025 16:56:19 +0500 Subject: [PATCH 50/77] lint --- types/src/v0/impls/stake_table.rs | 1 - utils/src/stake_table.rs | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/types/src/v0/impls/stake_table.rs b/types/src/v0/impls/stake_table.rs index 0ba7f1ddae..081b4a7c19 100644 --- a/types/src/v0/impls/stake_table.rs +++ b/types/src/v0/impls/stake_table.rs @@ -511,7 +511,6 @@ impl Membership for EpochCommittees { epoch: Epoch, block_header: Header, ) -> Option> { - // TODO: (abdul) fix fetching from contracts // so that order of l1 events match with the update let address = self.contract_address?; diff --git a/utils/src/stake_table.rs b/utils/src/stake_table.rs index 414991f654..ca20a8680f 100644 --- a/utils/src/stake_table.rs +++ b/utils/src/stake_table.rs @@ -14,7 +14,9 @@ use ethers::{ }; use hotshot::types::BLSPubKey; use hotshot_contract_adapter::stake_table::{bls_jf_to_sol, NodeInfoJf}; -use hotshot_types::{network::PeerConfigKeys, traits::signature_key::StakeTableEntryType, HotShotConfig}; +use hotshot_types::{ + network::PeerConfigKeys, traits::signature_key::StakeTableEntryType, HotShotConfig, +}; use url::Url; use std::{fs, path::Path, sync::Arc, time::Duration}; From c47a77eec697c8b79ce9b0ba1854a3016bc06db0 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 18 Feb 2025 17:05:18 +0500 Subject: [PATCH 51/77] fix env --- .github/workflows/test.yml | 2 +- .gitignore | 5 ++++- client/Cargo.toml | 2 +- data/initial_stake_table.toml | 30 ------------------------------ process-compose.yaml | 3 +-- 5 files changed, 7 insertions(+), 35 deletions(-) delete mode 100644 data/initial_stake_table.toml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f778253c44..8c93588ff5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -199,7 +199,7 @@ jobs: compose: "-f process-compose.yaml -D" - version: 03 env: - ESPRESSO_SEQUENCER_GENESIS_FILE=data/genesis/demo-pos-base.toml + ESPRESSO_SEQUENCER_GENESIS_FILE=data/genesis/demo-pos.toml compose: "-f process-compose.yaml -D" - version: 99 compose: "-f process-compose.yaml -f process-compose-mp.yml -D" diff --git a/.gitignore b/.gitignore index 17dd3e5e9a..0645c7ceda 100644 --- a/.gitignore +++ b/.gitignore @@ -56,4 +56,7 @@ contracts/broadcast/*/11155111/ docs/ # Autogen files -.vscode/ \ No newline at end of file +.vscode/ + +# initial stake table +data/initial_stake_table.toml \ No newline at end of file diff --git a/client/Cargo.toml b/client/Cargo.toml index c38a613844..9fe84cf055 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -10,9 +10,9 @@ anyhow = { workspace = true } espresso-types = { path = "../types" } ethers = { workspace = true } futures = { workspace = true } +hotshot-types = { workspace = true } jf-merkle-tree = { workspace = true } surf-disco = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } vbs = { workspace = true } -hotshot-types = { workspace = true } \ No newline at end of file diff --git a/data/initial_stake_table.toml b/data/initial_stake_table.toml deleted file mode 100644 index 57bbdffc68..0000000000 --- a/data/initial_stake_table.toml +++ /dev/null @@ -1,30 +0,0 @@ -[[new_stakers]] -stake_table_key = "BLS_VER_KEY~bQszS-QKYvUij2g20VqS8asttGSb95NrTu2PUj0uMh1CBUxNy1FqyPDjZqB29M7ZbjWqj79QkEOWkpga84AmDYUeTuWmy-0P1AdKHD3ehc-dKvei78BDj5USwXPJiDUlCxvYs_9rWYhagaq-5_LXENr78xel17spftNd5MA1Mw5U" -state_ver_key = "SCHNORR_VER_KEY~ibJCbfPOhDoURqiGLe683TDJ_KOLQCx8_Hdq43dOviSuL6WJJ_2mARKO3xA2k5zpXE3iiq4_z7mzvA-V1VXvIWw" -da = true -stake = 1 - -[[new_stakers]] -stake_table_key = "BLS_VER_KEY~4zQnaCOFJ7m95OjxeNls0QOOwWbz4rfxaL3NwmN2zSdnf8t5Nw_dfmMHq05ee8jCegw6Bn5T8inmrnGGAsQJMMWLv77nd7FJziz2ViAbXg-XGGF7o4HyzELCmypDOIYF3X2UWferFE_n72ZX0iQkUhOvYZZ7cfXToXxRTtb_mwRR" -state_ver_key = "SCHNORR_VER_KEY~lNCMqH5qLthH5OXxW_Z25tLXJUqmzzhsuQ6oVuaPWhtRPmgIKSqcBoJTaEbmGZL2VfTyQNguaoQL4U_4tCA_HmI" -da = true -stake = 1 - -[[new_stakers]] -stake_table_key = "BLS_VER_KEY~IBRoz_Q1EXvcm1pNZcmVlyYZU8hZ7qmy337ePAjEMhz8Hl2q8vWPFOd3BaLwgRS1UzAPW3z4E-XIgRDGcRBTAMZX9b_0lKYjlyTlNF2EZfNnKmvv-xJ0yurkfjiveeYEsD2l5d8q_rJJbH1iZdXy-yPEbwI0SIvQfwdlcaKw9po4" -state_ver_key = "SCHNORR_VER_KEY~nkFKzpLhJAafJ3LBkY_0h9OzxSyTu95Z029EUFPO4QNkeUo6DHQGTTVjxmprTA5H8jRSn73i0slJvig6dZ5kLX4" -da = true -stake = 1 - -[[new_stakers]] -stake_table_key = "BLS_VER_KEY~rO2PIjyY30HGfapFcloFe3mNDKMIFi6JlOLkH5ZWBSYoRm5fE2-Rm6Lp3EvmAcB5r7KFJ0c1Uor308x78r04EY_sfjcsDCWt7RSJdL4cJoD_4fSTCv_bisO8k98hs_8BtqQt8BHlPeJohpUXvcfnK8suXJETiJ6Er97pfxRbzgAL" -state_ver_key = "SCHNORR_VER_KEY~NwYhzlWarlZHxTNvChWuf74O3fP7zIt5NdC7V8gV6w2W92JOBDkrNmKQeMGxMUke-G5HHxUjHlZEWr1m1xLjEaI" -da = false -stake = 1 - - -[[new_stakers]] -stake_table_key = "BLS_VER_KEY~r6b-Cwzp-b3czlt0MHmYPJIow5kMsXbrNmZsLSYg9RV49oCCO4WEeCRFR02x9bqLCa_sgNFMrIeNdEa11qNiBAohApYFIvrSa-zP5QGj3xbZaMOCrshxYit6E2TR-XsWvv6gjOrypmugjyTAth-iqQzTboSfmO9DD1-gjJIdCaD7" -state_ver_key = "SCHNORR_VER_KEY~qMfMj1c1hRVTnugvz3MKNnVC5JA9jvZcV3ZCLL_J4Ap-u0i6ulGWveTk3OOelZj2-kd_WD5ojtYGWV1jHx9wCaA" -da = true -stake = 1 \ No newline at end of file diff --git a/process-compose.yaml b/process-compose.yaml index 6699a4fe90..7baa6767c4 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -5,8 +5,7 @@ environment: - ESPRESSO_SEQUENCER_ORCHESTRATOR_URL=http://localhost:$ESPRESSO_ORCHESTRATOR_PORT - ESPRESSO_SEQUENCER_URL=http://localhost:$ESPRESSO_SEQUENCER_API_PORT - ESPRESSO_SEQUENCER_L1_PROVIDER=http://localhost:$ESPRESSO_SEQUENCER_L1_PORT - # - ESPRESSO_SEQUENCER_GENESIS_FILE=$DEMO_GENESIS_FILE - # - ESPRESSO_BUILDER_GENESIS_FILE=$DEMO_GENESIS_FILE + - ESPRESSO_BUILDER_GENESIS_FILE=$ESPRESSO_SEQUENCER_GENESIS_FILE - ESPRESSO_SEQUENCER_INITIAL_PERMISSIONED_STAKE_TABLE_PATH=data/initial_stake_table.toml - ESPRESSO_STATE_RELAY_SERVER_URL=http://localhost:$ESPRESSO_STATE_RELAY_SERVER_PORT - QUERY_SERVICE_URI=http://localhost:$ESPRESSO_SEQUENCER1_API_PORT/v0/ From ab1cef68d74cdff477e47d3646a703f586652f75 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 18 Feb 2025 18:02:17 +0500 Subject: [PATCH 52/77] features for sqlite --- justfile | 8 ++++---- sequencer-sqlite/Cargo.toml | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/justfile b/justfile index 84b4493845..fa6b733edb 100644 --- a/justfile +++ b/justfile @@ -20,18 +20,18 @@ lint: cargo clippy --workspace --features testing --all-targets -- -D warnings cargo clippy --workspace --all-targets --manifest-path sequencer-sqlite/Cargo.toml -- -D warnings -build profile="test": +build profile="test" features="": #!/usr/bin/env bash set -euxo pipefail # Use the same target dir for both `build` invocations export CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-target} - cargo build --profile {{profile}} - cargo build --profile {{profile}} --manifest-path ./sequencer-sqlite/Cargo.toml + cargo build --profile {{profile}} {{features}} + cargo build --profile {{profile}} --manifest-path ./sequencer-sqlite/Cargo.toml {{features}} demo-native-mp *args: build scripts/demo-native -f process-compose.yaml -f process-compose-mp.yml {{args}} -demo-native-pos *args: build +demo-native-pos *args: (build "test" "--features fee,pos") ESPRESSO_SEQUENCER_GENESIS_FILE=data/genesis/demo-pos.toml scripts/demo-native -f process-compose.yaml {{args}} demo-native-pos-base *args: build diff --git a/sequencer-sqlite/Cargo.toml b/sequencer-sqlite/Cargo.toml index 5c6c1e6d58..189225244f 100644 --- a/sequencer-sqlite/Cargo.toml +++ b/sequencer-sqlite/Cargo.toml @@ -7,7 +7,9 @@ version = "0.1.0" edition = "2021" [features] -default = ["embedded-db"] +fee = ["sequencer/fee"] +pos = ["sequencer/pos"] +default = ["embedded-db", "pos"] sqlite-unbundled = ["sequencer/sqlite-unbundled"] embedded-db = ["sequencer/embedded-db"] From fc990f30677f71f5d02142684880cfcf3a617d68 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 18 Feb 2025 18:20:46 +0500 Subject: [PATCH 53/77] revert add_epoch_root() --- types/src/v0/impls/stake_table.rs | 35 +++++++------------------------ 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/types/src/v0/impls/stake_table.rs b/types/src/v0/impls/stake_table.rs index 081b4a7c19..990e5b6142 100644 --- a/types/src/v0/impls/stake_table.rs +++ b/types/src/v0/impls/stake_table.rs @@ -511,38 +511,17 @@ impl Membership for EpochCommittees { epoch: Epoch, block_header: Header, ) -> Option> { - // TODO: (abdul) fix fetching from contracts - // so that order of l1 events match with the update let address = self.contract_address?; - let genesis_st = self.state(&Some(Epoch::genesis())).unwrap().clone(); - let st = self - .l1_client + self.l1_client .get_stake_table(address.to_alloy(), block_header.l1_head()) .await - .ok(); - - let epoch_0_st = genesis_st.stake_table; - let epoch0_da = genesis_st.da_members; - - let sss = st.clone().unwrap(); - let contract_st = sss.stake_table; - let contract_da = sss.da_members; - - tracing::warn!("epoch0 st= {epoch_0_st:?}"); - tracing::warn!("contract st= {contract_st:?}"); - - tracing::warn!("epoch0 da= {contract_da:?}"); - tracing::warn!("contact da= {epoch0_da:?}"); - - let stake_tables = StakeTables { - stake_table: epoch_0_st.into(), - da_members: epoch0_da.into(), - }; - - Some(Box::new(move |committee: &mut Self| { - let _ = committee.update_stake_table(epoch, stake_tables); - })) + .ok() + .map(|stake_table| -> Box { + Box::new(move |committee: &mut Self| { + let _ = committee.update_stake_table(epoch, stake_table); + }) + }) } } #[cfg(test)] From 270af288b5de5c4cf1f47a80430c4fbe115d4246 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 18 Feb 2025 22:54:16 +0500 Subject: [PATCH 54/77] add epoch versions to builder main() --- builder/src/bin/permissionless-builder.rs | 28 +++++++++++++------ .../src/bin/marketplace-builder.rs | 25 ++++++++++++----- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/builder/src/bin/permissionless-builder.rs b/builder/src/bin/permissionless-builder.rs index 1ff329399e..771e5308eb 100644 --- a/builder/src/bin/permissionless-builder.rs +++ b/builder/src/bin/permissionless-builder.rs @@ -2,10 +2,7 @@ use std::{num::NonZeroUsize, path::PathBuf, time::Duration}; use builder::non_permissioned::{build_instance_state, BuilderConfig}; use clap::Parser; -use espresso_types::{ - eth_signature_key::EthKeyPair, parse_duration, FeeVersion, MarketplaceVersion, - SequencerVersions, V0_0, -}; +use espresso_types::{eth_signature_key::EthKeyPair, parse_duration, SequencerVersions}; use futures::future::pending; use hotshot::traits::ValidatedState; use hotshot_types::{ @@ -120,12 +117,25 @@ async fn main() -> anyhow::Result<()> { let upgrade = genesis.upgrade_version; match (base, upgrade) { - (FeeVersion::VERSION, MarketplaceVersion::VERSION) => { - run::>(genesis, opt).await + (espresso_types::FeeVersion::VERSION, espresso_types::EpochVersion::VERSION) => { + run::>( + genesis, opt + ) + .await + } + (espresso_types::EpochVersion::VERSION, _) => { + run::>( + genesis, opt + // Specifying V0_0 disables upgrades + ) + .await } - (FeeVersion::VERSION, _) => run::>(genesis, opt).await, - (MarketplaceVersion::VERSION, _) => { - run::>(genesis, opt).await + // TODO change `fee` to `pos` + (espresso_types::FeeVersion::VERSION, espresso_types::MarketplaceVersion::VERSION) => { + run::>( + genesis, opt + ) + .await } _ => panic!( "Invalid base ({base}) and upgrade ({upgrade}) versions specified in the toml file." diff --git a/marketplace-builder/src/bin/marketplace-builder.rs b/marketplace-builder/src/bin/marketplace-builder.rs index c23cf6a1b3..4091a41dce 100644 --- a/marketplace-builder/src/bin/marketplace-builder.rs +++ b/marketplace-builder/src/bin/marketplace-builder.rs @@ -2,8 +2,7 @@ use std::{num::NonZeroUsize, path::PathBuf, time::Duration}; use clap::Parser; use espresso_types::{ - eth_signature_key::EthKeyPair, parse_duration, FeeAmount, FeeVersion, MarketplaceVersion, - NamespaceId, SequencerVersions, V0_0, + eth_signature_key::EthKeyPair, parse_duration, FeeAmount, NamespaceId, SequencerVersions, }; use futures::future::pending; use hotshot::helpers::initialize_logging; @@ -127,12 +126,24 @@ async fn main() -> anyhow::Result<()> { let upgrade = genesis.upgrade_version; match (base, upgrade) { - (FeeVersion::VERSION, MarketplaceVersion::VERSION) => { - run::>(genesis, opt).await + (espresso_types::FeeVersion::VERSION, espresso_types::EpochVersion::VERSION) => { + run::>( + genesis, opt + ) + .await } - (FeeVersion::VERSION, _) => run::>(genesis, opt).await, - (MarketplaceVersion::VERSION, _) => { - run::>(genesis, opt).await + (espresso_types::EpochVersion::VERSION, _) => { + run::>( + genesis, opt + // Specifying V0_0 disables upgrades + ) + .await + } + (espresso_types::FeeVersion::VERSION, espresso_types::MarketplaceVersion::VERSION) => { + run::>( + genesis, opt + ) + .await } _ => panic!( "Invalid base ({base}) and upgrade ({upgrade}) versions specified in the toml file." From 3f3ea7c9152b67d8147da0cc18f4651b76644a9c Mon Sep 17 00:00:00 2001 From: lukaszrzasik Date: Tue, 18 Feb 2025 22:29:41 +0100 Subject: [PATCH 55/77] Lr/epoch integration test (#2639) --- types/src/v0/impls/state.rs | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/types/src/v0/impls/state.rs b/types/src/v0/impls/state.rs index f192a74274..a5ab7cfec0 100644 --- a/types/src/v0/impls/state.rs +++ b/types/src/v0/impls/state.rs @@ -909,29 +909,6 @@ impl HotShotState for ValidatedState { version: Version, view_number: u64, ) -> Result<(Self, Self::Delta), Self::Error> { - // During epoch transition, hotshot propagates the same block again - // we should totally skip this block, and return the same validated state - // This block will have the same parent block height - - tracing::debug!( - "parent_height={} proposed_height={}", - parent_leaf.height(), - proposed_header.height(), - ); - - if proposed_header.height() % instance.epoch_height == 0 - && parent_leaf.height() == proposed_header.height() - { - tracing::error!( - "skipping block.. parent_height={} proposed_height={} epoch_height={}", - parent_leaf.height(), - proposed_header.height(), - instance.epoch_height, - ); - - return Ok((self.clone(), Delta::default())); - } - // Unwrapping here is okay as we retry in a loop //so we should either get a validated state or until hotshot cancels the task let (validated_state, delta) = self From 389d5ae323b6815a9bf6a4cf189d972871d28b92 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Wed, 19 Feb 2025 19:02:18 +0500 Subject: [PATCH 56/77] change view settings for pos upgrade --- data/genesis/demo-pos.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/genesis/demo-pos.toml b/data/genesis/demo-pos.toml index b18e972383..3f99ded1e1 100644 --- a/data/genesis/demo-pos.toml +++ b/data/genesis/demo-pos.toml @@ -6,7 +6,7 @@ capacity = 10 [chain_config] chain_id = 999999999 -base_fee = '0 wei' +base_fee = '1 wei' max_block_size = '1mb' fee_recipient = '0x0000000000000000000000000000000000000000' fee_contract = '0xa15bb66138824a1c7167f5e85b957d04dd34e468' @@ -19,8 +19,8 @@ number = 0 [[upgrade]] version = "0.3" -start_proposing_view = 10 -stop_proposing_view = 60 +start_proposing_view = 65 +stop_proposing_view = 120 [upgrade.epoch] [upgrade.epoch.chain_config] From 54d79a3d0aa264e862676c34c94406c0fd37a5b1 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Wed, 19 Feb 2025 20:15:10 +0500 Subject: [PATCH 57/77] more fixes --- Cargo.lock | 2 + client/src/lib.rs | 4 +- sequencer-sqlite/Cargo.lock | 1 + sequencer/src/api.rs | 9 ++++ sequencer/src/api/data_source.rs | 2 + sequencer/src/api/endpoints.rs | 8 ++++ tests/Cargo.toml | 3 +- tests/common/mod.rs | 72 ++++++++++++++++++++++++++++++- tests/smoke.rs | 19 +++++++- tests/upgrades.rs | 66 +--------------------------- types/Cargo.toml | 1 + types/src/v0/impls/stake_table.rs | 68 ++++++++++++++--------------- 12 files changed, 150 insertions(+), 105 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da64fa74a5..eb7579c8c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3705,6 +3705,7 @@ dependencies = [ "hotshot-contract-adapter", "hotshot-query-service", "hotshot-types", + "indexmap 2.7.0", "itertools 0.12.1", "jf-merkle-tree", "jf-utils", @@ -11071,6 +11072,7 @@ dependencies = [ "sequencer-utils", "surf-disco", "tokio", + "tracing", "vbs", ] diff --git a/client/src/lib.rs b/client/src/lib.rs index aee2500074..13a54afce4 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -121,9 +121,9 @@ impl SequencerClient { Ok(balance) } - pub async fn current_epoch(&self) -> anyhow::Result { + pub async fn current_epoch(&self) -> anyhow::Result> { self.0 - .get::("node/current_epoch") + .get::>("node/current_epoch") .send() .await .context("getting epoch value") diff --git a/sequencer-sqlite/Cargo.lock b/sequencer-sqlite/Cargo.lock index 1ce8cf9bba..21ebf4a571 100644 --- a/sequencer-sqlite/Cargo.lock +++ b/sequencer-sqlite/Cargo.lock @@ -3516,6 +3516,7 @@ dependencies = [ "hotshot-contract-adapter", "hotshot-query-service", "hotshot-types", + "indexmap 2.7.0", "itertools 0.12.1", "jf-merkle-tree", "jf-utils", diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index ec9cca931d..8a59189e88 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -177,6 +177,11 @@ impl, D: Sync, V: Versions, P: SequencerPersistence> ) -> Vec::SignatureKey>> { self.as_ref().get_stake_table_current().await } + + /// Get the stake table for the current epoch if not provided + async fn get_current_epoch(&self) -> Option<::Epoch> { + self.as_ref().get_current_epoch().await + } } impl, V: Versions, P: SequencerPersistence> @@ -205,6 +210,10 @@ impl, V: Versions, P: SequencerPersistence> self.get_stake_table(epoch).await } + + async fn get_current_epoch(&self) -> Option<::Epoch> { + self.consensus().await.read().await.cur_epoch().await + } } impl, V: Versions, P: SequencerPersistence> SubmitDataSource diff --git a/sequencer/src/api/data_source.rs b/sequencer/src/api/data_source.rs index e3703d3874..52754e2548 100644 --- a/sequencer/src/api/data_source.rs +++ b/sequencer/src/api/data_source.rs @@ -127,6 +127,8 @@ pub(crate) trait StakeTableDataSource { fn get_stake_table_current( &self, ) -> impl Send + Future>>; + + fn get_current_epoch(&self) -> impl Send + Future>; } pub(crate) trait CatchupDataSource: Sync { diff --git a/sequencer/src/api/endpoints.rs b/sequencer/src/api/endpoints.rs index ed661026e8..b6d7fc35db 100644 --- a/sequencer/src/api/endpoints.rs +++ b/sequencer/src/api/endpoints.rs @@ -216,6 +216,14 @@ where .await) } .boxed() + })? + .at("current_epoch", |_, state| { + async move { + Ok(state + .read(|state| state.get_current_epoch().boxed()) + .await) + } + .boxed() })?; Ok(api) diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 09ae39b76c..69e0e5bef8 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -17,7 +17,8 @@ espresso-types = { path = "../types", features = ["testing"] } ethers = { workspace = true } futures = { workspace = true } reqwest = { workspace = true, features = ["json"] } +sequencer-utils = { path = "../utils" } surf-disco = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } vbs = { workspace = true } -sequencer-utils = { path = "../utils" } \ No newline at end of file diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 22b206cfbe..cea9bb2904 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,13 +1,19 @@ -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context, Result}; use client::SequencerClient; use espresso_types::{FeeAmount, FeeVersion, MarketplaceVersion}; use ethers::prelude::*; use futures::future::join_all; +use std::path::Path; use std::{fmt, str::FromStr, time::Duration}; use surf_disco::Url; use tokio::time::{sleep, timeout}; use vbs::version::StaticVersionType; +use dotenvy::var; +use sequencer_utils::stake_table::{ + update_stake_table, PermissionedStakeTableUpdate, StakerIdentity, +}; + const L1_PROVIDER_RETRY_INTERVAL: Duration = Duration::from_secs(1); // TODO add to .env const RECIPIENT_ADDRESS: &str = "0x0000000000000000000000000000000000000000"; @@ -277,3 +283,67 @@ async fn wait_for_service(url: Url, interval: u64, timeout_duration: u64) -> Res .await .map_err(|e| anyhow!("Wait for service, timeout: ({}) {}", url, e))? } + +pub async fn test_stake_table_update(clients: Vec) -> Result<()> { + /* + EPOCH V3 + */ + + let l1_port = var("ESPRESSO_SEQUENCER_L1_PORT")?; + let account_index = var("ESPRESSO_DEPLOYER_ACCOUNT_INDEX")?; + let contract_address = var("ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS")?; + let initial_stake_table_path = var("ESPRESSO_SEQUENCER_INITIAL_PERMISSIONED_STAKE_TABLE_PATH")?; + + let permissioned_stake_table = + PermissionedStakeTableUpdate::from_toml_file(Path::new(&initial_stake_table_path))?; + + // initial stake table has 5 new stakers + + let new_stakers = permissioned_stake_table.new_stakers; + //lets remove one + let staker_removed = new_stakers[0].clone(); + + let st_with_one_removed = PermissionedStakeTableUpdate::new( + vec![], + vec![StakerIdentity { + stake_table_key: staker_removed.stake_table_key.clone(), + }], + ); + let client = clients[0].clone(); + + let epoch_before_update = client.current_epoch().await?.context("curr epoch")?; + tracing::warn!("current_epoch={epoch_before_update:?}"); + update_stake_table( + format!("http://localhost:{l1_port}").parse()?, + Duration::from_secs(7), + "test test test test test test test test test test test junk".to_string(), + account_index.parse()?, + contract_address.parse()?, + st_with_one_removed, + ) + .await?; + + loop { + sleep(Duration::from_secs(10)).await; + let epoch = clients[0].current_epoch().await?.context("curr epoch")?; + tracing::info!("current_epoch={epoch:?}"); + if epoch > epoch_before_update + 6 { + let stake_table = client.stake_table(epoch).await?; + tracing::info!("stake_table={stake_table:?}"); + assert_eq!(stake_table.len(), 4); + + assert!( + stake_table + .iter() + .all(|st| st.stake_key != staker_removed.stake_table_key), + "Entry for {} already exists in the stake table", + staker_removed.stake_table_key + ); + + break; + } + } + // TODO: randomize this test + + Ok(()) +} diff --git a/tests/smoke.rs b/tests/smoke.rs index 69ea3be272..7177a90151 100644 --- a/tests/smoke.rs +++ b/tests/smoke.rs @@ -1,6 +1,7 @@ -use crate::common::TestConfig; -use anyhow::Result; +use crate::common::{test_stake_table_update, TestConfig}; +use anyhow::{Context, Result}; use futures::StreamExt; +use sequencer_utils::test_utils::setup_test; use std::time::Instant; /// We allow for no change in state across this many consecutive iterations. @@ -10,6 +11,7 @@ const MAX_TXNS_NOT_INCREMENTING: u8 = 5; #[tokio::test(flavor = "multi_thread")] async fn test_smoke() -> Result<()> { + setup_test(); let start = Instant::now(); dotenvy::dotenv()?; @@ -78,5 +80,18 @@ async fn test_smoke() -> Result<()> { last = new; } + + let epoch = testing + .espresso + .current_epoch() + .await? + .context("curr epoch")?; + + tracing::info!("epoch before stake table update {epoch:?}"); + + if epoch > 1 { + tracing::info!("testing stake table update"); + test_stake_table_update(testing.sequencer_clients).await?; + } Ok(()) } diff --git a/tests/upgrades.rs b/tests/upgrades.rs index 93e61bf347..d074ee35ac 100644 --- a/tests/upgrades.rs +++ b/tests/upgrades.rs @@ -1,13 +1,8 @@ -use std::{path::Path, time::Duration}; - -use crate::common::TestConfig; +use crate::common::{test_stake_table_update, TestConfig}; use anyhow::Result; use client::SequencerClient; -use dotenvy::var; use espresso_types::{EpochVersion, FeeVersion, MarketplaceVersion}; use futures::{future::join_all, StreamExt}; -use sequencer_utils::stake_table::{update_stake_table, PermissionedStakeTableUpdate}; -use tokio::time::sleep; use vbs::version::{StaticVersionType, Version}; const SEQUENCER_BLOCKS_TIMEOUT: u64 = 200; @@ -108,62 +103,3 @@ async fn test_blocks_production(clients: Vec, from: u64, num: u Ok(()) } - -async fn test_stake_table_update(clients: Vec) -> Result<()> { - /* - EPOCH V3 - */ - - let rpc_url = var("ESPRESSO_SEQUENCER_L1_PROVIDER")?; - let account_index = var("ESPRESSO_DEPLOYER_ACCOUNT_INDEX")?; - let contract_address = var("ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS")?; - let initial_stake_table_path = var("ESPRESSO_SEQUENCER_INITIAL_PERMISSIONED_STAKE_TABLE_PATH")?; - - let permissioned_stake_table = - PermissionedStakeTableUpdate::from_toml_file(Path::new(&initial_stake_table_path))?; - - // initial stake table has 5 new stakers - - let new_stakers = permissioned_stake_table.new_stakers; - //lets remove one - let staker_removed = new_stakers[0].clone(); - - let st_with_one_removed = - PermissionedStakeTableUpdate::new(vec![staker_removed.clone()], vec![]); - let client = clients[0].clone(); - - let epoch_before_update = client.current_epoch().await?; - - update_stake_table( - rpc_url.parse()?, - Duration::from_secs(7), - "test test test test test test test test test test test junk".to_string(), - account_index.parse()?, - contract_address.parse()?, - st_with_one_removed, - ) - .await?; - - loop { - sleep(Duration::from_secs(10)).await; - let epoch = clients[0].current_epoch().await?; - - if epoch > epoch_before_update { - let stake_table = client.stake_table(epoch).await?; - assert_eq!(stake_table.len(), 4); - - assert!( - stake_table - .iter() - .all(|st| st.stake_key != staker_removed.stake_table_key), - "Entry for {} already exists in the stake table", - staker_removed.stake_table_key - ); - - break; - } - } - // TODO: randomize this test - - Ok(()) -} diff --git a/types/Cargo.toml b/types/Cargo.toml index 40c10d304a..7828de6db7 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -31,6 +31,7 @@ hotshot = { workspace = true } hotshot-contract-adapter = { workspace = true } hotshot-query-service = { workspace = true } hotshot-types = { workspace = true } +indexmap = "2.7" itertools = { workspace = true } jf-merkle-tree = { workspace = true } jf-utils = { workspace = true } # TODO temporary: used only for test_rng() diff --git a/types/src/v0/impls/stake_table.rs b/types/src/v0/impls/stake_table.rs index 990e5b6142..458e5bdcbb 100644 --- a/types/src/v0/impls/stake_table.rs +++ b/types/src/v0/impls/stake_table.rs @@ -19,7 +19,7 @@ use hotshot_types::{ }, PeerConfig, }; -use itertools::Itertools; +use indexmap::IndexMap; use std::{ cmp::max, collections::{BTreeSet, HashMap}, @@ -49,43 +49,36 @@ impl StakeTables { /// should not significantly affect performance to fetch all events and /// perform the computation in this functions once per epoch. pub fn from_l1_events(updates: Vec) -> Self { - let changes_per_node = updates - .into_iter() - .flat_map(|event| { - event - .removed - .into_iter() - .map(|key| StakeTableChange::Remove(bls_alloy_to_jf(key))) - .chain( - event - .added - .into_iter() - .map(|node_info| StakeTableChange::Add(node_info.into())), - ) - }) - .group_by(|change| change.key()); + let mut index_map = IndexMap::new(); - // If the last event for a stakers is `Added` the staker is currently - // staking, if the last event is removed or (or the staker is not present) - // they are not staking. - let currently_staking = changes_per_node - .into_iter() - .map(|(_pub_key, deltas)| deltas.last().expect("deltas non-empty").clone()) - .filter_map(|change| match change { - StakeTableChange::Add(node_info) => Some(node_info), - StakeTableChange::Remove(_) => None, - }); - - let mut consensus_stake_table: Vec> = vec![]; - let mut da_members: Vec> = vec![]; - for node in currently_staking { - consensus_stake_table.push(node.clone().into()); - if node.da { - da_members.push(node.into()); + for event in updates { + for key in event.removed { + let change = StakeTableChange::Remove(bls_alloy_to_jf(key)); + index_map.insert(change.key(), change); + } + for node_info in event.added { + let change = StakeTableChange::Add(node_info.into()); + index_map.insert(change.key(), change); } } - Self::new(consensus_stake_table.into(), da_members.into()) + let mut da_members = Vec::new(); + let mut stake_table = Vec::new(); + + for change in index_map.values() { + if let StakeTableChange::Add(node_info_jf) = change { + let entry: StakeTableEntry = node_info_jf.clone().into(); + stake_table.push(entry.clone()); + if change.is_da() { + da_members.push(entry); + } + } + } + + tracing::error!("DA={da_members:?}"); + tracing::error!("ST={stake_table:?}"); + + Self::new(stake_table.into(), da_members.into()) } } @@ -121,6 +114,13 @@ impl StakeTableChange { StakeTableChange::Remove(key) => *key, } } + + pub(crate) fn is_da(&self) -> bool { + match self { + StakeTableChange::Add(node_info) => node_info.da, + StakeTableChange::Remove(_) => false, + } + } } /// Holds Stake table and da stake From ced65a19c4e05aa858968cfe6ad8d8ff9301e877 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Wed, 19 Feb 2025 21:24:03 +0500 Subject: [PATCH 58/77] remove bid_recipient from config --- data/genesis/demo-pos.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/data/genesis/demo-pos.toml b/data/genesis/demo-pos.toml index 3f99ded1e1..db0e4db50c 100644 --- a/data/genesis/demo-pos.toml +++ b/data/genesis/demo-pos.toml @@ -28,6 +28,5 @@ chain_id = 999999999 max_block_size = '1mb' base_fee = '1 wei' fee_recipient = "0x0000000000000000000000000000000000000000" -bid_recipient = "0x0000000000000000000000000000000000000000" fee_contract = "0xa15bb66138824a1c7167f5e85b957d04dd34e468" stake_table_contract = "0xb19b36b1456e65e3a6d514d3f715f204bd59f431" \ No newline at end of file From 88d4a2e6d11df406c07cc816dcd086c2b9256f0d Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Wed, 19 Feb 2025 22:16:58 +0500 Subject: [PATCH 59/77] preload epoch staketables --- types/src/v0/impls/stake_table.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/types/src/v0/impls/stake_table.rs b/types/src/v0/impls/stake_table.rs index 458e5bdcbb..adeb989967 100644 --- a/types/src/v0/impls/stake_table.rs +++ b/types/src/v0/impls/stake_table.rs @@ -189,8 +189,10 @@ impl EpochCommittees { }; self.state.insert(epoch, committee.clone()); - self.state.insert(epoch + 1, committee.clone()); - self.state.insert(epoch + 2, committee.clone()); + for i in epoch.u64()..=epoch.u64() + 15 { + self.state.insert(EpochNumber::new(i + 1), committee.clone()); + } + committee } @@ -246,7 +248,7 @@ impl EpochCommittees { // TODO: remove this, workaround for hotshot asking for stake tables from epoch 1 and 2 let mut map = HashMap::new(); - for epoch in Epoch::genesis().u64()..=10 { + for epoch in Epoch::genesis().u64()..=50 { map.insert(Epoch::new(epoch), members.clone()); } From e3050130e99593da754abbdde6813787c3c2e6e6 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Thu, 20 Feb 2025 17:05:17 +0500 Subject: [PATCH 60/77] get contract address from chain config and do catchup --- data/genesis/demo-pos.toml | 4 +- sequencer/src/api/endpoints.rs | 7 +- types/src/v0/impls/stake_table.rs | 123 +++++++++++++----------------- 3 files changed, 55 insertions(+), 79 deletions(-) diff --git a/data/genesis/demo-pos.toml b/data/genesis/demo-pos.toml index db0e4db50c..cf0b37596a 100644 --- a/data/genesis/demo-pos.toml +++ b/data/genesis/demo-pos.toml @@ -19,8 +19,8 @@ number = 0 [[upgrade]] version = "0.3" -start_proposing_view = 65 -stop_proposing_view = 120 +start_proposing_view = 10 +stop_proposing_view = 50 [upgrade.epoch] [upgrade.epoch.chain_config] diff --git a/sequencer/src/api/endpoints.rs b/sequencer/src/api/endpoints.rs index b6d7fc35db..aac4d4f281 100644 --- a/sequencer/src/api/endpoints.rs +++ b/sequencer/src/api/endpoints.rs @@ -218,12 +218,7 @@ where .boxed() })? .at("current_epoch", |_, state| { - async move { - Ok(state - .read(|state| state.get_current_epoch().boxed()) - .await) - } - .boxed() + async move { Ok(state.read(|state| state.get_current_epoch().boxed()).await) }.boxed() })?; Ok(api) diff --git a/types/src/v0/impls/stake_table.rs b/types/src/v0/impls/stake_table.rs index adeb989967..d1b7f7c585 100644 --- a/types/src/v0/impls/stake_table.rs +++ b/types/src/v0/impls/stake_table.rs @@ -1,11 +1,14 @@ use super::{ + traits::StateCatchup, v0_3::{DAMembers, StakeTable, StakeTables}, + v0_99::ChainConfig, Header, L1Client, NodeState, PubKey, SeqTypes, }; use async_trait::async_trait; +use committable::Committable; use contract_bindings_alloy::permissionedstaketable::PermissionedStakeTable::StakersUpdated; -use ethers::types::{Address, U256}; +use ethers::types::U256; use ethers_conv::ToAlloy; use hotshot::types::{BLSPubKey, SignatureKey as _}; use hotshot_contract_adapter::stake_table::{bls_alloy_to_jf, NodeInfoJf}; @@ -24,12 +27,10 @@ use std::{ cmp::max, collections::{BTreeSet, HashMap}, num::NonZeroU64, - str::FromStr, + sync::Arc, }; use thiserror::Error; -use url::Url; - type Epoch = ::Epoch; impl StakeTables { @@ -82,7 +83,7 @@ impl StakeTables { } } -#[derive(Clone, Debug)] +#[derive(derive_more::Debug, Clone)] /// Type to describe DA and Stake memberships pub struct EpochCommittees { /// Committee used when we're in pre-epoch state @@ -97,8 +98,9 @@ pub struct EpochCommittees { /// L1 provider l1_client: L1Client, - /// Address of Stake Table Contract - contract_address: Option
, + chain_config: ChainConfig, + #[debug("{}", peers.name())] + pub peers: Arc, } #[derive(Debug, Clone, PartialEq)] @@ -189,9 +191,8 @@ impl EpochCommittees { }; self.state.insert(epoch, committee.clone()); - for i in epoch.u64()..=epoch.u64() + 15 { - self.state.insert(EpochNumber::new(i + 1), committee.clone()); - } + self.state.insert(epoch + 1, committee.clone()); + self.state.insert(epoch + 2, committee.clone()); committee } @@ -252,14 +253,13 @@ impl EpochCommittees { map.insert(Epoch::new(epoch), members.clone()); } - let address = instance_state.chain_config.stake_table_contract; - Self { non_epoch_committee: members, state: map, _epoch_size: epoch_size, l1_client: instance_state.l1_client.clone(), - contract_address: address, + chain_config: instance_state.chain_config, + peers: instance_state.peers.clone(), } } @@ -284,65 +284,11 @@ impl Membership for EpochCommittees { fn new( // TODO remove `new` from trait and remove this fn as well. // https://github.com/EspressoSystems/HotShot/commit/fcb7d54a4443e29d643b3bbc53761856aef4de8b - committee_members: Vec>, - da_members: Vec>, + _committee_members: Vec>, + _da_members: Vec>, ) -> Self { - // For each eligible leader, get the stake table entry - let eligible_leaders: Vec<_> = committee_members - .iter() - .map(|member| member.stake_table_entry.clone()) - .filter(|entry| entry.stake() > U256::zero()) - .collect(); - - // For each member, get the stake table entry - let stake_table: Vec<_> = committee_members - .iter() - .map(|member| member.stake_table_entry.clone()) - .filter(|entry| entry.stake() > U256::zero()) - .collect(); - - // For each member, get the stake table entry - let da_members: Vec<_> = da_members - .iter() - .map(|member| member.stake_table_entry.clone()) - .filter(|entry| entry.stake() > U256::zero()) - .collect(); - - // Index the stake table by public key - let indexed_stake_table: HashMap = stake_table - .iter() - .map(|entry| (PubKey::public_key(entry), entry.clone())) - .collect(); - - // Index the stake table by public key - let indexed_da_members: HashMap = da_members - .iter() - .map(|entry| (PubKey::public_key(entry), entry.clone())) - .collect(); - - let members = Committee { - eligible_leaders, - stake_table, - da_members, - indexed_stake_table, - indexed_da_members, - }; - - let mut map = HashMap::new(); - map.insert(Epoch::genesis(), members.clone()); - // TODO: remove this, workaround for hotshot asking for stake tables from epoch 1 - map.insert(Epoch::genesis() + 1u64, members.clone()); - - Self { - non_epoch_committee: members, - state: map, - _epoch_size: 12, - l1_client: L1Client::new(vec![Url::from_str("http:://ab.b").unwrap()]) - .expect("Failed to create L1 client"), - contract_address: None, - } + panic!("EpochCommittees::new() called. This function has been replaced with new_stake()"); } - /// Get the stake table for the current view fn stake_table(&self, epoch: Option) -> Vec> { let st = if let Some(st) = self.state(&epoch) { @@ -513,7 +459,17 @@ impl Membership for EpochCommittees { epoch: Epoch, block_header: Header, ) -> Option> { - let address = self.contract_address?; + let chain_config = get_chain_config(self.chain_config, &self.peers, &block_header) + .await + .ok()?; + + let contract_address = chain_config.stake_table_contract; + + if contract_address.is_none() { + tracing::error!("No stake table contract address found in Chain config"); + } + + let address = contract_address?; self.l1_client .get_stake_table(address.to_alloy(), block_header.l1_head()) @@ -526,6 +482,31 @@ impl Membership for EpochCommittees { }) } } + +pub(crate) async fn get_chain_config( + chain_config: ChainConfig, + peers: &impl StateCatchup, + header: &Header, +) -> anyhow::Result { + let header_cf = header.chain_config(); + if chain_config.commit() == header_cf.commit() { + return Ok(chain_config); + } + + let cf = match header_cf.resolve() { + Some(cf) => cf, + None => peers + .fetch_chain_config(header_cf.commit()) + .await + .map_err(|err| { + tracing::error!("failed to get chain_config from peers. err: {err:?}"); + err + })?, + }; + + Ok(cf) +} + #[cfg(test)] mod tests { use contract_bindings_alloy::permissionedstaketable::PermissionedStakeTable::NodeInfo; From e6558a41134df368fd33954869e1975af6f0b882 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Mon, 24 Feb 2025 15:33:16 +0500 Subject: [PATCH 61/77] set epoch height in genesis file --- data/genesis/demo-pos-base.toml | 2 +- data/genesis/demo-pos.toml | 1 + data/v3/messages.bin | Bin 7404 -> 7405 bytes data/v3/messages.json | 3 +- sequencer/src/api.rs | 63 ++++++++++++++++---------------- sequencer/src/genesis.rs | 1 + sequencer/src/lib.rs | 5 ++- sequencer/src/restart_tests.rs | 1 + sequencer/src/run.rs | 1 + 9 files changed, 41 insertions(+), 36 deletions(-) diff --git a/data/genesis/demo-pos-base.toml b/data/genesis/demo-pos-base.toml index 8d174799a1..d909148ef6 100644 --- a/data/genesis/demo-pos-base.toml +++ b/data/genesis/demo-pos-base.toml @@ -1,5 +1,6 @@ base_version = "0.3" upgrade_version = "0.3" +epoch_height = 10 [stake_table] capacity = 10 @@ -9,7 +10,6 @@ chain_id = 999999999 max_block_size = '1mb' base_fee = '1 wei' fee_recipient = "0x0000000000000000000000000000000000000000" -# bid_recipient = "0x0000000000000000000000000000000000000000" fee_contract = "0xa15bb66138824a1c7167f5e85b957d04dd34e468" stake_table_contract = "0xb19b36b1456e65e3a6d514d3f715f204bd59f431" diff --git a/data/genesis/demo-pos.toml b/data/genesis/demo-pos.toml index cf0b37596a..6414cab0d2 100644 --- a/data/genesis/demo-pos.toml +++ b/data/genesis/demo-pos.toml @@ -1,5 +1,6 @@ base_version = "0.2" upgrade_version = "0.3" +epoch_height = 10 [stake_table] capacity = 10 diff --git a/data/v3/messages.bin b/data/v3/messages.bin index b07e615a29611c080714f53adc33395bd6aefc38..ba47afc646765941eada03bcf3a1038dc049a5bc 100644 GIT binary patch delta 20 ccmaE3`POoSDdXhl60(zziz{v}Vyuz|0ATY7E&u=k delta 20 ccmaEB`NndCDdXgq5;Budh%0U`W~`D10AUXZF#rGn diff --git a/data/v3/messages.json b/data/v3/messages.json index fcf77f9592..eed2d4964f 100644 --- a/data/v3/messages.json +++ b/data/v3/messages.json @@ -18,7 +18,8 @@ "chain_id": "35353", "fee_contract": null, "fee_recipient": "0x0000000000000000000000000000000000000000", - "max_block_size": "30720" + "max_block_size": "30720", + "stake_table_contract": null } } }, diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 8a59189e88..208e41b8f5 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -1576,9 +1576,8 @@ mod test { use espresso_types::{ traits::NullEventConsumer, v0_1::{UpgradeMode, ViewBasedUpgrade}, - BackoffParams, FeeAccount, FeeAmount, FeeVersion, Header, MarketplaceVersion, - MockSequencerVersions, SequencerVersions, TimeBasedUpgrade, Timestamp, Upgrade, - UpgradeType, ValidatedState, + BackoffParams, FeeAccount, FeeAmount, Header, MarketplaceVersion, MockSequencerVersions, + SequencerVersions, TimeBasedUpgrade, Timestamp, Upgrade, UpgradeType, ValidatedState, }; use ethers::utils::Anvil; use futures::{ @@ -2138,35 +2137,35 @@ mod test { handle.abort(); } - #[tokio::test(flavor = "multi_thread")] - async fn test_pos_upgrade_view_based() { - setup_test(); - - let mut upgrades = std::collections::BTreeMap::new(); - type MySequencerVersions = SequencerVersions; - - let mode = UpgradeMode::View(ViewBasedUpgrade { - start_voting_view: None, - stop_voting_view: None, - start_proposing_view: 1, - stop_proposing_view: 10, - }); - - let upgrade_type = UpgradeType::Epoch { - chain_config: ChainConfig { - max_block_size: 500.into(), - base_fee: 2.into(), - stake_table_contract: Some(Default::default()), - ..Default::default() - }, - }; - - upgrades.insert( - ::Upgrade::VERSION, - Upgrade { mode, upgrade_type }, - ); - test_upgrade_helper::(upgrades, MySequencerVersions::new()).await; - } + // #[tokio::test(flavor = "multi_thread")] + // async fn test_pos_upgrade_view_based() { + // setup_test(); + + // let mut upgrades = std::collections::BTreeMap::new(); + // type MySequencerVersions = SequencerVersions; + + // let mode = UpgradeMode::View(ViewBasedUpgrade { + // start_voting_view: None, + // stop_voting_view: None, + // start_proposing_view: 1, + // stop_proposing_view: 10, + // }); + + // let upgrade_type = UpgradeType::Epoch { + // chain_config: ChainConfig { + // max_block_size: 500.into(), + // base_fee: 2.into(), + // stake_table_contract: Some(Default::default()), + // ..Default::default() + // }, + // }; + + // upgrades.insert( + // ::Upgrade::VERSION, + // Upgrade { mode, upgrade_type }, + // ); + // test_upgrade_helper::(upgrades, MySequencerVersions::new()).await; + // } #[tokio::test(flavor = "multi_thread")] async fn test_marketplace_upgrade_view_based() { diff --git a/sequencer/src/genesis.rs b/sequencer/src/genesis.rs index b1ddf0838e..790a8d6193 100644 --- a/sequencer/src/genesis.rs +++ b/sequencer/src/genesis.rs @@ -53,6 +53,7 @@ pub struct Genesis { pub base_version: Version, #[serde(with = "version_ser")] pub upgrade_version: Version, + pub epoch_height: Option, pub chain_config: ChainConfig, pub stake_table: StakeTableConfig, #[serde(default)] diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 79eb77c133..5f8c6553fd 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -363,8 +363,9 @@ pub async fn init_node( upgrade.set_hotshot_config_parameters(&mut network_config.config); } - //todo(abdul): get from genesis file - network_config.config.epoch_height = 10; + let epoch_height = genesis.epoch_height.unwrap_or_default(); + tracing::info!("setting epoch height={epoch_height:?}"); + network_config.config.epoch_height = epoch_height; // If the `Libp2p` bootstrap nodes were supplied via the command line, override those // present in the config file. diff --git a/sequencer/src/restart_tests.rs b/sequencer/src/restart_tests.rs index 7631120994..4c6c5a82c5 100755 --- a/sequencer/src/restart_tests.rs +++ b/sequencer/src/restart_tests.rs @@ -543,6 +543,7 @@ impl TestNetwork { upgrades: Default::default(), base_version: Version { major: 0, minor: 1 }, upgrade_version: Version { major: 0, minor: 2 }, + epoch_height: None, // Start with a funded account, so we can test catchup after restart. accounts: [(builder_account(), 1000000000.into())] diff --git a/sequencer/src/run.rs b/sequencer/src/run.rs index 53aa25291a..568716a268 100644 --- a/sequencer/src/run.rs +++ b/sequencer/src/run.rs @@ -293,6 +293,7 @@ mod test { upgrades: Default::default(), base_version: Version { major: 0, minor: 1 }, upgrade_version: Version { major: 0, minor: 2 }, + epoch_height: None, }; genesis.to_file(&genesis_file).unwrap(); From c1b1e223915d0beaf04ac5ff4b487460cdb84eda Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Mon, 24 Feb 2025 15:58:43 +0500 Subject: [PATCH 62/77] lockfile --- Cargo.lock | 2 +- sequencer-sqlite/Cargo.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2eb50a70f..9afec16fa3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3727,7 +3727,7 @@ dependencies = [ "hotshot-contract-adapter", "hotshot-query-service", "hotshot-types", - "indexmap 2.7.0", + "indexmap 2.7.1", "itertools 0.12.1", "jf-merkle-tree", "jf-utils", diff --git a/sequencer-sqlite/Cargo.lock b/sequencer-sqlite/Cargo.lock index fc7b0bbacb..c5ca0e1b2f 100644 --- a/sequencer-sqlite/Cargo.lock +++ b/sequencer-sqlite/Cargo.lock @@ -3553,7 +3553,7 @@ dependencies = [ "hotshot-contract-adapter", "hotshot-query-service", "hotshot-types", - "indexmap 2.7.0", + "indexmap 2.7.1", "itertools 0.12.1", "jf-merkle-tree", "jf-utils", From e69b5f2c125f73ead5788c004ad724ee705c3c4f Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Mon, 24 Feb 2025 20:17:47 +0500 Subject: [PATCH 63/77] load initial stake table from config --- Cargo.toml | 8 +++ client/src/lib.rs | 19 ++++- flake.nix | 8 ++- hotshot-types/src/data.rs | 2 +- sequencer/api/node.toml | 9 +++ sequencer/src/api.rs | 38 ++++++++++ sequencer/src/api/data_source.rs | 9 +++ sequencer/src/api/endpoints.rs | 26 +++++++ .../bin/update-permissioned-stake-table.rs | 71 +++++++++++++++++-- sequencer/src/lib.rs | 7 -- utils/src/stake_table.rs | 24 +------ 11 files changed, 182 insertions(+), 39 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7c7d9278b3..b80c04588a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -224,6 +224,14 @@ paste = "1.0" rand = "0.8.5" time = "0.3" trait-set = "0.3.0" + +[profile.dev] +# No optimizations +opt-level = 0 +# Skip compiling the debug information. +debug = false +# Skip linking symbols. +strip = true [profile.test] opt-level = 1 [profile.test.package.tests] diff --git a/client/src/lib.rs b/client/src/lib.rs index 13a54afce4..703fe639e4 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -10,6 +10,7 @@ use jf_merkle_tree::{ use std::time::Duration; use surf_disco::{ error::ClientError, + http::convert::DeserializeOwned, socket::{Connection, Unsupported}, Url, }; @@ -134,7 +135,23 @@ impl SequencerClient { .get::<_>(&format!("node/stake-table/{epoch}")) .send() .await - .context("getting epoch value") + .context("getting stake table") + } + + pub async fn da_members(&self, epoch: u64) -> anyhow::Result>> { + self.0 + .get::<_>(&format!("node/stake-table/da/{epoch}")) + .send() + .await + .context("getting da stake table") + } + + pub async fn config(&self) -> anyhow::Result { + self.0 + .get::(&format!("config/hotshot")) + .send() + .await + .context("getting hotshot config") } } diff --git a/flake.nix b/flake.nix index be4c42ecc4..c4e74ad5fe 100644 --- a/flake.nix +++ b/flake.nix @@ -67,7 +67,13 @@ solhintPkg { inherit (prev) buildNpmPackage fetchFromGitHub; }; }) - + # The mold linker is around 50% faster on Linux than the default linker. + # This overlays a mkShell that is configured to use mold on Linux. + (final: prev: prev.lib.optionalAttrs prev.stdenv.isLinux { + mkShell = prev.mkShell.override { + stdenv = prev.stdenvAdapters.useMoldLinker prev.clangStdenv; + }; + }) ]; pkgs = import nixpkgs { inherit system overlays; }; crossShell = { config }: diff --git a/hotshot-types/src/data.rs b/hotshot-types/src/data.rs index ed8e7ceba2..1227b43fc5 100644 --- a/hotshot-types/src/data.rs +++ b/hotshot-types/src/data.rs @@ -138,7 +138,7 @@ impl_u64_wrapper!(EpochNumber); impl EpochNumber { /// Create a genesis number (1) #[allow(dead_code)] - fn genesis() -> Self { + pub fn genesis() -> Self { Self(1) } } diff --git a/sequencer/api/node.toml b/sequencer/api/node.toml index d2025500f7..1ad3d4a05c 100644 --- a/sequencer/api/node.toml +++ b/sequencer/api/node.toml @@ -2,11 +2,20 @@ PATH = ["stake-table/current"] DOC = "Get the stake table for the current epoch" +[route.da_members_current] +PATH = ["stake-table/da/current"] +DOC = "Get the stake table da members for the current epoch" + [route.stake_table] PATH = ["stake-table/:epoch_number"] ":epoch_number" = "Integer" DOC = "Get the stake table for the given epoch" +[route.da_members] +PATH = ["stake-table/da/:epoch_number"] +":epoch_number" = "Integer" +DOC = "Get the stake table da members for the given epoch" + [route.current_epoch] PATH = ["current_epoch"] DOC = "Get the current epoch" \ No newline at end of file diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 208e41b8f5..4c26e3a660 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -171,6 +171,14 @@ impl, D: Sync, V: Versions, P: SequencerPersistence> self.as_ref().get_stake_table(epoch).await } + /// Get the stake table for a given epoch + async fn get_da_members( + &self, + epoch: Option<::Epoch>, + ) -> Vec::SignatureKey>> { + self.as_ref().get_da_members(epoch).await + } + /// Get the stake table for the current epoch if not provided async fn get_stake_table_current( &self, @@ -178,6 +186,13 @@ impl, D: Sync, V: Versions, P: SequencerPersistence> self.as_ref().get_stake_table_current().await } + /// Get the stake table for the current epoch if not provided + async fn get_da_members_current( + &self, + ) -> Vec::SignatureKey>> { + self.as_ref().get_da_members_current().await + } + /// Get the stake table for the current epoch if not provided async fn get_current_epoch(&self) -> Option<::Epoch> { self.as_ref().get_current_epoch().await @@ -214,6 +229,29 @@ impl, V: Versions, P: SequencerPersistence> async fn get_current_epoch(&self) -> Option<::Epoch> { self.consensus().await.read().await.cur_epoch().await } + + async fn get_da_members( + &self, + epoch: Option<::Epoch>, + ) -> Vec::SignatureKey>> { + self.consensus() + .await + .read() + .await + .memberships + .read() + .await + .da_stake_table(epoch) + } + + /// Get the stake table for the current epoch if not provided + async fn get_da_members_current( + &self, + ) -> Vec::SignatureKey>> { + let epoch = self.consensus().await.read().await.cur_epoch().await; + + self.get_da_members(epoch).await + } } impl, V: Versions, P: SequencerPersistence> SubmitDataSource diff --git a/sequencer/src/api/data_source.rs b/sequencer/src/api/data_source.rs index 52754e2548..a3372a754b 100644 --- a/sequencer/src/api/data_source.rs +++ b/sequencer/src/api/data_source.rs @@ -129,6 +129,15 @@ pub(crate) trait StakeTableDataSource { ) -> impl Send + Future>>; fn get_current_epoch(&self) -> impl Send + Future>; + + fn get_da_members( + &self, + epoch: Option<::Epoch>, + ) -> impl Send + Future::SignatureKey>>>; + /// Get the stake table for the current epoch if not provided + fn get_da_members_current( + &self, + ) -> impl Send + Future::SignatureKey>>>; } pub(crate) trait CatchupDataSource: Sync { diff --git a/sequencer/src/api/endpoints.rs b/sequencer/src/api/endpoints.rs index aac4d4f281..95c4971030 100644 --- a/sequencer/src/api/endpoints.rs +++ b/sequencer/src/api/endpoints.rs @@ -219,6 +219,32 @@ where })? .at("current_epoch", |_, state| { async move { Ok(state.read(|state| state.get_current_epoch().boxed()).await) }.boxed() + })? + .at("da_members", |req, state| { + async move { + // Try to get the epoch from the request. If this fails, error + // as it was probably a mistake + let epoch = req + .opt_integer_param("epoch_number") + .map_err(|_| hotshot_query_service::node::Error::Custom { + message: "Epoch number is required".to_string(), + status: StatusCode::BAD_REQUEST, + })? + .map(EpochNumber::new); + + Ok(state + .read(|state| state.get_da_members(epoch).boxed()) + .await) + } + .boxed() + })? + .at("da_members_current", |_, state| { + async move { + Ok(state + .read(|state| state.get_da_members_current().boxed()) + .await) + } + .boxed() })?; Ok(api) diff --git a/sequencer/src/bin/update-permissioned-stake-table.rs b/sequencer/src/bin/update-permissioned-stake-table.rs index 890080ef3e..bae660784c 100644 --- a/sequencer/src/bin/update-permissioned-stake-table.rs +++ b/sequencer/src/bin/update-permissioned-stake-table.rs @@ -1,12 +1,16 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use clap::Parser; +use client::SequencerClient; use espresso_types::parse_duration; use ethers::types::Address; +use hotshot_types::{network::PeerConfigKeys, traits::signature_key::StakeTableEntryType}; +use sequencer::api::data_source::PublicHotShotConfig; use sequencer_utils::{ logging, stake_table::{update_stake_table, PermissionedStakeTableUpdate}, }; use std::{path::PathBuf, time::Duration}; + use url::Url; #[derive(Debug, Clone, Parser)] @@ -78,7 +82,28 @@ struct Options { env = "ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_UPDATE_TOML_PATH", verbatim_doc_comment )] - update_toml_path: PathBuf, + update_toml_path: Option, + /// Flag to update the contract with the initial stake table. + /// + /// This stake table is fetched directly from hotshot config, and is pre-epoch stake table + + #[clap( + long, + short, + env = "ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_INITIAL", + default_value_t = false + )] + initial: bool, + + /// Peer nodes use to fetch missing state + #[clap( + long, + env = "ESPRESSO_SEQUENCER_STATE_PEERS", + value_delimiter = ',', + conflicts_with = "update_toml_path" + )] + pub state_peers: Option>, + #[clap(flatten)] logging: logging::Config, } @@ -88,10 +113,44 @@ async fn main() -> Result<()> { let opts = Options::parse(); opts.logging.init(); - let path = opts.update_toml_path; + let mut update: Option = None; + + if opts.initial { + let peers = opts.state_peers.context("No state peers found")?; + let clients: Vec = peers.into_iter().map(SequencerClient::new).collect(); + + for client in &clients { + tracing::warn!("calling config endpoint of {client:?}"); + + match client.config::().await { + Ok(config) => { + let hotshot = config.into_hotshot_config(); + let st = hotshot.known_nodes_with_stake; + let da_nodes = hotshot.known_da_nodes; + + let new_stakers = st + .into_iter() + .map(|s| PeerConfigKeys { + stake_table_key: s.stake_table_entry.stake_key.clone(), + state_ver_key: s.state_ver_key.clone(), + stake: s.stake_table_entry.stake().as_u64(), + da: da_nodes.contains(&s), + }) + .collect(); - tracing::error!("updating stake table from path: {path:?}"); - let update = PermissionedStakeTableUpdate::from_toml_file(&path)?; + update = Some(PermissionedStakeTableUpdate::new(new_stakers, Vec::new())); + break; + } + Err(e) => { + tracing::warn!("Failed to fetch config from sequencer: {e}"); + } + }; + } + } else { + let path = opts.update_toml_path.context("No update path found")?; + tracing::error!("updating stake table from path: {path:?}"); + update = Some(PermissionedStakeTableUpdate::from_toml_file(&path)?); + }; update_stake_table( opts.rpc_url, @@ -99,7 +158,7 @@ async fn main() -> Result<()> { opts.mnemonic, opts.account_index, opts.contract_address, - update, + update.unwrap(), ) .await?; diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 5f8c6553fd..d79d16ae13 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -23,7 +23,6 @@ use espresso_types::{ use ethers_conv::ToAlloy; use genesis::L1Finalized; use proposal_fetcher::ProposalFetcherConfig; -use sequencer_utils::stake_table::PermissionedStakeTableUpdate; use std::sync::Arc; use tokio::select; // Should move `STAKE_TABLE_CAPACITY` in the sequencer repo when we have variate stake table support @@ -496,12 +495,6 @@ pub async fn init_node( network_config.config.epoch_height, ); - // save initial stake table into toml file - // this will be helpful to load it into contract - PermissionedStakeTableUpdate::save_initial_stake_table_from_hotshot_config( - network_config.config.clone(), - )?; - // Initialize the Libp2p network let network = { let p2p_network = Libp2pNetwork::from_config( diff --git a/utils/src/stake_table.rs b/utils/src/stake_table.rs index ca20a8680f..6c7fc4bfb6 100644 --- a/utils/src/stake_table.rs +++ b/utils/src/stake_table.rs @@ -14,9 +14,7 @@ use ethers::{ }; use hotshot::types::BLSPubKey; use hotshot_contract_adapter::stake_table::{bls_jf_to_sol, NodeInfoJf}; -use hotshot_types::{ - network::PeerConfigKeys, traits::signature_key::StakeTableEntryType, HotShotConfig, -}; +use hotshot_types::network::PeerConfigKeys; use url::Url; use std::{fs, path::Path, sync::Arc, time::Duration}; @@ -132,26 +130,6 @@ impl PermissionedStakeTableUpdate { }) .collect() } - - pub fn save_initial_stake_table_from_hotshot_config( - config: HotShotConfig, - ) -> anyhow::Result<()> { - let committee_members = config.known_nodes_with_stake.clone(); - let known_da_nodes = config.known_da_nodes.clone().clone(); - let members = committee_members - .into_iter() - .map(|m| PeerConfigKeys { - stake_table_key: m.stake_table_entry.public_key(), - state_ver_key: m.state_ver_key.clone(), - stake: m.stake_table_entry.stake().as_u64(), - da: known_da_nodes.contains(&m), - }) - .collect(); - - Self::new(members, vec![]).to_toml_file(Path::new("data/initial_stake_table.toml"))?; - - Ok(()) - } } pub async fn update_stake_table( From 8bf12798737052714898df697647fe5a891671b6 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Mon, 24 Feb 2025 20:40:30 +0500 Subject: [PATCH 64/77] fix update binary --- process-compose.yaml | 2 +- sequencer/src/api/data_source.rs | 4 + .../bin/update-permissioned-stake-table.rs | 90 +++++++++---------- 3 files changed, 47 insertions(+), 49 deletions(-) diff --git a/process-compose.yaml b/process-compose.yaml index 7baa6767c4..33c7ab8d87 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -117,7 +117,7 @@ processes: command: update-permissioned-stake-table environment: - ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS - - ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_UPDATE_TOML_PATH=data/initial_stake_table.toml + - ESPRESSO_SEQUENCER_STATE_PEERS=http://localhost:$ESPRESSO_SEQUENCER_API_PORT depends_on: deploy-prover-contracts: condition: process_completed diff --git a/sequencer/src/api/data_source.rs b/sequencer/src/api/data_source.rs index a3372a754b..b8afaa5b95 100644 --- a/sequencer/src/api/data_source.rs +++ b/sequencer/src/api/data_source.rs @@ -435,6 +435,10 @@ impl PublicNetworkConfig { public_keys: Vec::new(), }) } + + pub fn hotshot_config(&self) -> PublicHotShotConfig { + self.config.clone() + } } #[cfg(any(test, feature = "testing"))] diff --git a/sequencer/src/bin/update-permissioned-stake-table.rs b/sequencer/src/bin/update-permissioned-stake-table.rs index bae660784c..a5a53f6647 100644 --- a/sequencer/src/bin/update-permissioned-stake-table.rs +++ b/sequencer/src/bin/update-permissioned-stake-table.rs @@ -4,7 +4,7 @@ use client::SequencerClient; use espresso_types::parse_duration; use ethers::types::Address; use hotshot_types::{network::PeerConfigKeys, traits::signature_key::StakeTableEntryType}; -use sequencer::api::data_source::PublicHotShotConfig; +use sequencer::api::data_source::PublicNetworkConfig; use sequencer_utils::{ logging, stake_table::{update_stake_table, PermissionedStakeTableUpdate}, @@ -83,19 +83,10 @@ struct Options { verbatim_doc_comment )] update_toml_path: Option, - /// Flag to update the contract with the initial stake table. - /// - /// This stake table is fetched directly from hotshot config, and is pre-epoch stake table - - #[clap( - long, - short, - env = "ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_INITIAL", - default_value_t = false - )] - initial: bool, - /// Peer nodes use to fetch missing state + /// Peers for fetching hotshot config + /// used to update the contract with the initial stake table. + /// This stake table is fetched directly from hotshot config, and is pre-epoch stake table #[clap( long, env = "ESPRESSO_SEQUENCER_STATE_PEERS", @@ -115,42 +106,45 @@ async fn main() -> Result<()> { let mut update: Option = None; - if opts.initial { - let peers = opts.state_peers.context("No state peers found")?; - let clients: Vec = peers.into_iter().map(SequencerClient::new).collect(); - - for client in &clients { - tracing::warn!("calling config endpoint of {client:?}"); - - match client.config::().await { - Ok(config) => { - let hotshot = config.into_hotshot_config(); - let st = hotshot.known_nodes_with_stake; - let da_nodes = hotshot.known_da_nodes; - - let new_stakers = st - .into_iter() - .map(|s| PeerConfigKeys { - stake_table_key: s.stake_table_entry.stake_key.clone(), - state_ver_key: s.state_ver_key.clone(), - stake: s.stake_table_entry.stake().as_u64(), - da: da_nodes.contains(&s), - }) - .collect(); - - update = Some(PermissionedStakeTableUpdate::new(new_stakers, Vec::new())); - break; - } - Err(e) => { - tracing::warn!("Failed to fetch config from sequencer: {e}"); - } - }; + match opts.update_toml_path { + Some(path) => { + tracing::error!("updating stake table from path: {path:?}"); + update = Some(PermissionedStakeTableUpdate::from_toml_file(&path)?); + } + None => { + let peers = opts.state_peers.context("No state peers found")?; + let clients: Vec = + peers.into_iter().map(SequencerClient::new).collect(); + + for client in &clients { + tracing::warn!("calling config endpoint of {client:?}"); + + match client.config::().await { + Ok(config) => { + let hotshot = config.hotshot_config().into_hotshot_config(); + let st = hotshot.known_nodes_with_stake; + let da_nodes = hotshot.known_da_nodes; + + let new_stakers = st + .into_iter() + .map(|s| PeerConfigKeys { + stake_table_key: s.stake_table_entry.stake_key.clone(), + state_ver_key: s.state_ver_key.clone(), + stake: s.stake_table_entry.stake().as_u64(), + da: da_nodes.contains(&s), + }) + .collect(); + + update = Some(PermissionedStakeTableUpdate::new(new_stakers, Vec::new())); + break; + } + Err(e) => { + tracing::warn!("Failed to fetch config from sequencer: {e}"); + } + }; + } } - } else { - let path = opts.update_toml_path.context("No update path found")?; - tracing::error!("updating stake table from path: {path:?}"); - update = Some(PermissionedStakeTableUpdate::from_toml_file(&path)?); - }; + } update_stake_table( opts.rpc_url, From 54c89d7b02683c7d842e2d9dff4cdea94e6615fb Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 25 Feb 2025 01:07:51 +0500 Subject: [PATCH 65/77] fix stake table update test --- client/src/lib.rs | 2 +- tests/common/mod.rs | 127 ++++++++++++++++++++++++--------------- tests/smoke.rs | 1 + utils/src/stake_table.rs | 14 +++-- 4 files changed, 90 insertions(+), 54 deletions(-) diff --git a/client/src/lib.rs b/client/src/lib.rs index 703fe639e4..f941d96be7 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -148,7 +148,7 @@ impl SequencerClient { pub async fn config(&self) -> anyhow::Result { self.0 - .get::(&format!("config/hotshot")) + .get::("config/hotshot") .send() .await .context("getting hotshot config") diff --git a/tests/common/mod.rs b/tests/common/mod.rs index cea9bb2904..3adff49579 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -2,8 +2,8 @@ use anyhow::{anyhow, Context, Result}; use client::SequencerClient; use espresso_types::{FeeAmount, FeeVersion, MarketplaceVersion}; use ethers::prelude::*; -use futures::future::join_all; -use std::path::Path; +use futures::future::{join_all, BoxFuture}; +use futures::FutureExt; use std::{fmt, str::FromStr, time::Duration}; use surf_disco::Url; use tokio::time::{sleep, timeout}; @@ -292,58 +292,91 @@ pub async fn test_stake_table_update(clients: Vec) -> Result<() let l1_port = var("ESPRESSO_SEQUENCER_L1_PORT")?; let account_index = var("ESPRESSO_DEPLOYER_ACCOUNT_INDEX")?; let contract_address = var("ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS")?; - let initial_stake_table_path = var("ESPRESSO_SEQUENCER_INITIAL_PERMISSIONED_STAKE_TABLE_PATH")?; - - let permissioned_stake_table = - PermissionedStakeTableUpdate::from_toml_file(Path::new(&initial_stake_table_path))?; - - // initial stake table has 5 new stakers + let client = clients[0].clone(); + // currently stake table update does not support DA node member changes - let new_stakers = permissioned_stake_table.new_stakers; - //lets remove one - let staker_removed = new_stakers[0].clone(); + let stake_table = client.stake_table(1).await?; + let da_members = client.da_members(1).await?; - let st_with_one_removed = PermissionedStakeTableUpdate::new( + // filtering out DA nodes + let stakers: Vec<_> = stake_table + .into_iter() + .filter(|x| !da_members.contains(x)) + .collect(); + + let assert_change = + move |u: PermissionedStakeTableUpdate| -> BoxFuture<'static, anyhow::Result<()>> { + async move { + let epoch_before_update = client.current_epoch().await?.context("curr epoch")?; + tracing::warn!("current_epoch={epoch_before_update:?}"); + + let current_stake_table = client.stake_table(epoch_before_update).await?; + + let removed = u.stakers_to_remove.len(); + let added = u.new_stakers.len(); + + update_stake_table( + format!("http://localhost:{l1_port}").parse()?, + Duration::from_secs(7), + "test test test test test test test test test test test junk".to_string(), + account_index.parse()?, + contract_address.parse()?, + u.clone(), + ) + .await?; + + loop { + sleep(Duration::from_secs(10)).await; + let epoch = client.current_epoch().await?.context("curr epoch")?; + tracing::info!("current_epoch={epoch:?}"); + if epoch > epoch_before_update + 2 { + let stake_table = client.stake_table(epoch).await?; + tracing::info!("stake_table={stake_table:?}"); + assert_eq!( + stake_table.len(), + current_stake_table.len() + added - removed + ); + + for added in &u.new_stakers { + assert!( + stake_table + .iter() + .any(|st| st.stake_key == added.stake_table_key), + "staker {} not found", + added.stake_table_key + ); + } + + for removed in &u.stakers_to_remove { + assert!( + stake_table + .iter() + .all(|st| st.stake_key != removed.stake_table_key), + "staker {} found", + removed.stake_table_key + ); + } + + break; + } + } + + anyhow::Result::<_>::Ok(()) + } + .boxed() + }; + let node = stakers[0].clone(); + let one_removed = PermissionedStakeTableUpdate::new( vec![], vec![StakerIdentity { - stake_table_key: staker_removed.stake_table_key.clone(), + stake_table_key: node.stake_key.clone(), }], ); - let client = clients[0].clone(); - let epoch_before_update = client.current_epoch().await?.context("curr epoch")?; - tracing::warn!("current_epoch={epoch_before_update:?}"); - update_stake_table( - format!("http://localhost:{l1_port}").parse()?, - Duration::from_secs(7), - "test test test test test test test test test test test junk".to_string(), - account_index.parse()?, - contract_address.parse()?, - st_with_one_removed, - ) - .await?; - - loop { - sleep(Duration::from_secs(10)).await; - let epoch = clients[0].current_epoch().await?.context("curr epoch")?; - tracing::info!("current_epoch={epoch:?}"); - if epoch > epoch_before_update + 6 { - let stake_table = client.stake_table(epoch).await?; - tracing::info!("stake_table={stake_table:?}"); - assert_eq!(stake_table.len(), 4); - - assert!( - stake_table - .iter() - .all(|st| st.stake_key != staker_removed.stake_table_key), - "Entry for {} already exists in the stake table", - staker_removed.stake_table_key - ); - - break; - } - } - // TODO: randomize this test + // remove one node + assert_change(one_removed) + .await + .expect("failed to remove one node"); Ok(()) } diff --git a/tests/smoke.rs b/tests/smoke.rs index 7177a90151..0a0a23cda3 100644 --- a/tests/smoke.rs +++ b/tests/smoke.rs @@ -89,6 +89,7 @@ async fn test_smoke() -> Result<()> { tracing::info!("epoch before stake table update {epoch:?}"); + // Check if epoch number is greater than Epoch::genesis() i.e 1 if epoch > 1 { tracing::info!("testing stake table update"); test_stake_table_update(testing.sequencer_clients).await?; diff --git a/utils/src/stake_table.rs b/utils/src/stake_table.rs index 6c7fc4bfb6..6cbb96c0d0 100644 --- a/utils/src/stake_table.rs +++ b/utils/src/stake_table.rs @@ -3,7 +3,7 @@ /// The initial stake table is passed to the permissioned stake table contract /// on deployment. use contract_bindings_ethers::permissioned_stake_table::{ - G2Point, NodeInfo, PermissionedStakeTable, + G2Point, NodeInfo, PermissionedStakeTable, PermissionedStakeTableErrors, }; use derive_more::derive::From; use ethers::{ @@ -19,6 +19,8 @@ use url::Url; use std::{fs, path::Path, sync::Arc, time::Duration}; +use crate::contract_send; + /// A stake table config stored in a file #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] #[serde(bound(deserialize = ""))] @@ -158,11 +160,11 @@ pub async fn update_stake_table( anyhow::bail!("No changes to update in the stake table"); } - let tx_receipt = contract - .update(update.stakers_to_remove(), update.new_stakers()) - .send() - .await? - .await?; + let (tx_receipt, _) = contract_send::<_, _, PermissionedStakeTableErrors>( + &contract.update(update.stakers_to_remove(), update.new_stakers()), + ) + .await?; + tracing::info!("Transaction receipt: {:?}", tx_receipt); Ok(()) } From 9bd10ef5f084f49687bb17371f75f470cefe432b Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 25 Feb 2025 01:17:48 +0500 Subject: [PATCH 66/77] clippy --- sequencer/src/bin/update-permissioned-stake-table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequencer/src/bin/update-permissioned-stake-table.rs b/sequencer/src/bin/update-permissioned-stake-table.rs index a5a53f6647..37adadba46 100644 --- a/sequencer/src/bin/update-permissioned-stake-table.rs +++ b/sequencer/src/bin/update-permissioned-stake-table.rs @@ -128,7 +128,7 @@ async fn main() -> Result<()> { let new_stakers = st .into_iter() .map(|s| PeerConfigKeys { - stake_table_key: s.stake_table_entry.stake_key.clone(), + stake_table_key: s.stake_table_entry.stake_key, state_ver_key: s.state_ver_key.clone(), stake: s.stake_table_entry.stake().as_u64(), da: da_nodes.contains(&s), From 51ec2436818a5a66f06731fc45aa7f3485b3ce1e Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 25 Feb 2025 02:19:30 +0500 Subject: [PATCH 67/77] remove l1_provider, mnemonic default values --- process-compose.yaml | 2 ++ .../src/bin/update-permissioned-stake-table.rs | 14 ++------------ tests/common/mod.rs | 2 +- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/process-compose.yaml b/process-compose.yaml index 33c7ab8d87..6a704feffd 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -118,6 +118,8 @@ processes: environment: - ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS - ESPRESSO_SEQUENCER_STATE_PEERS=http://localhost:$ESPRESSO_SEQUENCER_API_PORT + - ESPRESSO_SEQUENCER_ETH_MNEMONIC + - ESPRESSO_SEQUENCER_L1_PROVIDER depends_on: deploy-prover-contracts: condition: process_completed diff --git a/sequencer/src/bin/update-permissioned-stake-table.rs b/sequencer/src/bin/update-permissioned-stake-table.rs index 37adadba46..d0a3bac1c1 100644 --- a/sequencer/src/bin/update-permissioned-stake-table.rs +++ b/sequencer/src/bin/update-permissioned-stake-table.rs @@ -16,12 +16,7 @@ use url::Url; #[derive(Debug, Clone, Parser)] struct Options { /// RPC URL for the L1 provider. - #[clap( - short, - long, - env = "ESPRESSO_SEQUENCER_L1_PROVIDER", - default_value = "http://localhost:8545" - )] + #[clap(short, long, env = "ESPRESSO_SEQUENCER_L1_PROVIDER")] rpc_url: Url, /// Request rate when polling L1. @@ -37,12 +32,7 @@ struct Options { /// /// This wallet is used to deploy the contracts, so the account indicated by ACCOUNT_INDEX must /// be funded with with ETH. - #[clap( - long, - name = "MNEMONIC", - env = "ESPRESSO_SEQUENCER_ETH_MNEMONIC", - default_value = "test test test test test test test test test test test junk" - )] + #[clap(long, name = "MNEMONIC", env = "ESPRESSO_SEQUENCER_ETH_MNEMONIC")] mnemonic: String, /// Account index in the L1 wallet generated by MNEMONIC to use when deploying the contracts. diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 3adff49579..5bc18547aa 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -369,7 +369,7 @@ pub async fn test_stake_table_update(clients: Vec) -> Result<() let one_removed = PermissionedStakeTableUpdate::new( vec![], vec![StakerIdentity { - stake_table_key: node.stake_key.clone(), + stake_table_key: node.stake_key, }], ); From a0614377a33f06a209b2e35754f347855c4a2b3a Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 25 Feb 2025 17:49:40 +0500 Subject: [PATCH 68/77] use config endpoint for stake table updates --- .github/workflows/test.yml | 2 +- Cargo.lock | 2 ++ tests/Cargo.toml | 2 ++ tests/common/mod.rs | 74 ++++++++++++++++++++++++++++---------- tests/smoke.rs | 1 - 5 files changed, 61 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8c93588ff5..50eedd1211 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -193,7 +193,7 @@ jobs: needs: [build-test-bins, build-test-artifacts-postgres] strategy: matrix: - version: [03] + version: [02, 03, 99] include: - version: 02 compose: "-f process-compose.yaml -D" diff --git a/Cargo.lock b/Cargo.lock index 9afec16fa3..62bce33a43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11161,8 +11161,10 @@ dependencies = [ "espresso-types", "ethers", "futures", + "hotshot-types", "reqwest 0.12.12", "sequencer-utils", + "serde", "surf-disco", "tokio", "tracing", diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 69e0e5bef8..e33048117b 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -22,3 +22,5 @@ surf-disco = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } vbs = { workspace = true } +hotshot-types = { workspace = true } +serde = { workspace = true } \ No newline at end of file diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 5bc18547aa..99224034f3 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,10 +1,14 @@ use anyhow::{anyhow, Context, Result}; use client::SequencerClient; -use espresso_types::{FeeAmount, FeeVersion, MarketplaceVersion}; +use espresso_types::{FeeAmount, FeeVersion, MarketplaceVersion, PubKey}; use ethers::prelude::*; use futures::future::{join_all, BoxFuture}; use futures::FutureExt; +use hotshot_types::network::PeerConfigKeys; +use hotshot_types::traits::signature_key::StakeTableEntryType; +use hotshot_types::PeerConfig; use std::{fmt, str::FromStr, time::Duration}; +use surf_disco::http::convert::Deserialize; use surf_disco::Url; use tokio::time::{sleep, timeout}; use vbs::version::StaticVersionType; @@ -284,28 +288,22 @@ async fn wait_for_service(url: Url, interval: u64, timeout_duration: u64) -> Res .map_err(|e| anyhow!("Wait for service, timeout: ({}) {}", url, e))? } -pub async fn test_stake_table_update(clients: Vec) -> Result<()> { - /* - EPOCH V3 - */ +/* + EPOCH V3 +*/ +pub async fn test_stake_table_update(clients: Vec) -> Result<()> { let l1_port = var("ESPRESSO_SEQUENCER_L1_PORT")?; let account_index = var("ESPRESSO_DEPLOYER_ACCOUNT_INDEX")?; let contract_address = var("ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS")?; let client = clients[0].clone(); - // currently stake table update does not support DA node member changes - - let stake_table = client.stake_table(1).await?; - let da_members = client.da_members(1).await?; - - // filtering out DA nodes - let stakers: Vec<_> = stake_table - .into_iter() - .filter(|x| !da_members.contains(x)) - .collect(); let assert_change = - move |u: PermissionedStakeTableUpdate| -> BoxFuture<'static, anyhow::Result<()>> { + |u: PermissionedStakeTableUpdate| -> BoxFuture<'static, anyhow::Result<()>> { + let client = client.clone(); + let l1_port = l1_port.clone(); + let account_index = account_index.clone(); + let contract_address = contract_address.clone(); async move { let epoch_before_update = client.current_epoch().await?.context("curr epoch")?; tracing::warn!("current_epoch={epoch_before_update:?}"); @@ -365,11 +363,27 @@ pub async fn test_stake_table_update(clients: Vec) -> Result<() } .boxed() }; - let node = stakers[0].clone(); + + let config = client.config::().await?.config; + + // currently stake table update does not support DA node member changes + let stake_table = config.known_nodes_with_stake; + let da_members = config.known_da_nodes; + + // filtering out DA nodes + let non_da_stakers: Vec<_> = stake_table + .into_iter() + .filter(|x| !da_members.contains(x)) + .collect(); + + let node = non_da_stakers + .get(0) + .context("failed to get non DA node")? + .clone(); let one_removed = PermissionedStakeTableUpdate::new( vec![], vec![StakerIdentity { - stake_table_key: node.stake_key, + stake_table_key: node.stake_table_entry.stake_key.clone(), }], ); @@ -378,5 +392,29 @@ pub async fn test_stake_table_update(clients: Vec) -> Result<() .await .expect("failed to remove one node"); + // add back the removed node + let added = PermissionedStakeTableUpdate::new( + vec![PeerConfigKeys { + stake_table_key: *node.stake_table_entry.key(), + state_ver_key: node.state_ver_key, + stake: node.stake_table_entry.stake().as_u64(), + da: false, + }], + vec![], + ); + + assert_change(added).await.expect("failed to add a node"); + Ok(()) } + +#[derive(Debug, Deserialize)] +struct PublicHotShotConfig { + known_nodes_with_stake: Vec>, + known_da_nodes: Vec>, +} + +#[derive(Debug, Deserialize)] +struct PublicNetworkConfig { + config: PublicHotShotConfig, +} diff --git a/tests/smoke.rs b/tests/smoke.rs index 0a0a23cda3..e009fb3ba3 100644 --- a/tests/smoke.rs +++ b/tests/smoke.rs @@ -32,7 +32,6 @@ async fn test_smoke() -> Result<()> { let mut state_retries = 0; let mut txn_retries = 0; while (sub.next().await).is_some() { - dbg!("next"); let new = testing.test_state().await; println!("New State:{}", new); From fbbe05fae5a8f4dff2a3f6e202008c1422b80090 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 25 Feb 2025 18:30:48 +0500 Subject: [PATCH 69/77] lint --- tests/common/mod.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 99224034f3..f0abe66e0e 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -376,14 +376,11 @@ pub async fn test_stake_table_update(clients: Vec) -> Result<() .filter(|x| !da_members.contains(x)) .collect(); - let node = non_da_stakers - .get(0) - .context("failed to get non DA node")? - .clone(); + let node = non_da_stakers.first().context("no non da staker found")?; let one_removed = PermissionedStakeTableUpdate::new( vec![], vec![StakerIdentity { - stake_table_key: node.stake_table_entry.stake_key.clone(), + stake_table_key: node.stake_table_entry.stake_key, }], ); @@ -396,7 +393,7 @@ pub async fn test_stake_table_update(clients: Vec) -> Result<() let added = PermissionedStakeTableUpdate::new( vec![PeerConfigKeys { stake_table_key: *node.stake_table_entry.key(), - state_ver_key: node.state_ver_key, + state_ver_key: node.state_ver_key.clone(), stake: node.stake_table_entry.stake().as_u64(), da: false, }], From 3e645910821771bcb132a03b644a319efbaa1ec7 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 25 Feb 2025 18:52:38 +0500 Subject: [PATCH 70/77] use fee,pos,marketplace to build binaries for integration-test --- .github/workflows/test.yml | 4 ++-- tests/Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 50eedd1211..3737805559 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -94,8 +94,8 @@ jobs: - name: Build Bins run: | - cargo build --locked --profile test --bins - cargo build --manifest-path ./sequencer-sqlite/Cargo.toml --target-dir ./target + cargo build --features "fee,pos,marketplace" --locked --profile test --bins + cargo build --features "fee,pos,marketplace" --manifest-path ./sequencer-sqlite/Cargo.toml --target-dir ./target timeout-minutes: 60 - name: Upload archive to workflow diff --git a/tests/Cargo.toml b/tests/Cargo.toml index e33048117b..551f6f1f36 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -16,11 +16,11 @@ dotenvy = { workspace = true } espresso-types = { path = "../types", features = ["testing"] } ethers = { workspace = true } futures = { workspace = true } +hotshot-types = { workspace = true } reqwest = { workspace = true, features = ["json"] } sequencer-utils = { path = "../utils" } +serde = { workspace = true } surf-disco = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } vbs = { workspace = true } -hotshot-types = { workspace = true } -serde = { workspace = true } \ No newline at end of file From 6ee04ba7213c3ba3aaf5c3741ca74864d0a56f14 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 25 Feb 2025 19:00:49 +0500 Subject: [PATCH 71/77] fix docker-compose.yaml --- docker-compose.yaml | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 22e29f0e34..c9309f5fd7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -15,11 +15,10 @@ services: - "./geth-config/genesis-default.json:/genesis.json" - "./geth-config/test-jwt-secret.txt:/config/test-jwt-secret.txt" - deploy-sequencer-contracts: + deploy-fee-contract: image: ghcr.io/espressosystems/espresso-sequencer/deploy:main - command: deploy --only fee-contract,permissioned-stake-table + command: deploy --only fee-contract environment: - - ESPRESSO_SEQUENCER_INITIAL_PERMISSIONED_STAKE_TABLE_PATH=/data/initial_stake_table.toml - ESPRESSO_SEQUENCER_ETH_MULTISIG_ADDRESS - ESPRESSO_SEQUENCER_L1_PROVIDER - ESPRESSO_SEQUENCER_L1_POLLING_INTERVAL @@ -27,12 +26,24 @@ services: - RUST_LOG - RUST_LOG_FORMAT - ASYNC_STD_THREAD_COUNT - volumes: - - "./data/initial_stake_table.toml:/data/initial_stake_table.toml" depends_on: demo-l1-network: condition: service_healthy + deploy-stake-table-contract: + image: ghcr.io/espressosystems/espresso-sequencer/deploy:main + command: deploy --only permissioned-stake-table + environment: + - ESPRESSO_SEQUENCER_L1_PROVIDER + - ESPRESSO_SEQUENCER_L1_POLLING_INTERVAL + - ESPRESSO_DEPLOYER_ACCOUNT_INDEX + - RUST_LOG + - RUST_LOG_FORMAT + - ASYNC_STD_THREAD_COUNT + depends_on: + deploy-fee-contract: + condition: service_completed_successfully + deploy-prover-contracts: image: ghcr.io/espressosystems/espresso-sequencer/deploy:main command: deploy --use-mock-contract --only light-client @@ -54,7 +65,9 @@ services: sequencer0: condition: service_healthy # Make sure this doesn't start until the other contracts have been deployed, since we use the same mnemonic. - deploy-sequencer-contracts: + deploy-stake-table-contract: + condition: service_completed_successfully + deploy-fee-contract: condition: service_completed_successfully fund-builder: @@ -220,6 +233,19 @@ services: deploy-prover-contracts: condition: service_completed_successfully + + update-permissioned-stake-table: + image: ghcr.io/espressosystems/espresso-sequencer/update-permissioned-stake-table:main + environment: + - ESPRESSO_SEQUENCER_PERMISSIONED_STAKE_TABLE_ADDRESS + - ESPRESSO_SEQUENCER_STATE_PEERS=http://sequencer:$ESPRESSO_SEQUENCER_API_PORT + - ESPRESSO_SEQUENCER_ETH_MNEMONIC + - ESPRESSO_SEQUENCER_L1_PROVIDER + depends_on: + deploy-prover-contracts: + condition: service_completed_successfully + sequencer0: + condition: service_healthy sequencer0: image: ghcr.io/espressosystems/espresso-sequencer/sequencer:main ports: From a78a33dbb2fd559b03129e89d7323b54a68de340 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Tue, 25 Feb 2025 19:54:31 +0500 Subject: [PATCH 72/77] add marketplace feature to sqlite binary --- sequencer-sqlite/Cargo.toml | 1 + tests/upgrades.rs | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/sequencer-sqlite/Cargo.toml b/sequencer-sqlite/Cargo.toml index 189225244f..8ada2b8dde 100644 --- a/sequencer-sqlite/Cargo.toml +++ b/sequencer-sqlite/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" [features] fee = ["sequencer/fee"] pos = ["sequencer/pos"] +marketplace = ["sequencer/marketplace"] default = ["embedded-db", "pos"] sqlite-unbundled = ["sequencer/sqlite-unbundled"] embedded-db = ["sequencer/embedded-db"] diff --git a/tests/upgrades.rs b/tests/upgrades.rs index d074ee35ac..9c6ebafb9d 100644 --- a/tests/upgrades.rs +++ b/tests/upgrades.rs @@ -26,15 +26,19 @@ async fn test_upgrade() -> Result<()> { println!("Initial State:{}", initial); let clients = testing.sequencer_clients; - + let client = clients[0].clone(); let height = test_header_version(clients.clone(), base, upgrade).await?; // check that atleast 50 blocks are produced after the upgrade test_blocks_production(clients.clone(), height, 50).await?; if upgrade == EpochVersion::version() { - test_stake_table_update(clients).await?; + test_stake_table_update(clients.clone()).await?; } + let height = client.get_height().await?; + // check that atleast 50 blocks are produced after the stake table updates + test_blocks_production(clients.clone(), height, 50).await?; + // TODO assert transactions are incrementing Ok(()) } From 2fc136f382bee157c1638cf0fe62455ce4ec4337 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Wed, 26 Feb 2025 13:56:45 +0500 Subject: [PATCH 73/77] insert randomized committee --- docker-compose.yaml | 12 ++++++------ types/src/v0/impls/stake_table.rs | 14 +++++++++++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index c9309f5fd7..2e6a892065 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -85,7 +85,7 @@ services: - RUST_LOG - RUST_LOG_FORMAT depends_on: - deploy-sequencer-contracts: + deploy-stake-table-contract: condition: service_completed_successfully sequencer1: condition: service_healthy @@ -304,7 +304,7 @@ services: condition: service_healthy marshal-0: condition: service_healthy - deploy-sequencer-contracts: + deploy-stake-table-contract: condition: service_completed_successfully sequencer1: @@ -363,7 +363,7 @@ services: condition: service_healthy marshal-0: condition: service_healthy - deploy-sequencer-contracts: + deploy-stake-table-contract: condition: service_completed_successfully sequencer2: @@ -414,7 +414,7 @@ services: condition: service_healthy marshal-0: condition: service_healthy - deploy-sequencer-contracts: + deploy-stake-table-contract: condition: service_completed_successfully sequencer3: @@ -466,7 +466,7 @@ services: condition: service_healthy marshal-0: condition: service_healthy - deploy-sequencer-contracts: + deploy-stake-table-contract: condition: service_completed_successfully sequencer4: @@ -517,7 +517,7 @@ services: condition: service_healthy marshal-0: condition: service_healthy - deploy-sequencer-contracts: + deploy-stake-table-contract: condition: service_completed_successfully submit-transactions-public: diff --git a/types/src/v0/impls/stake_table.rs b/types/src/v0/impls/stake_table.rs index 0bb290947d..49d164badb 100644 --- a/types/src/v0/impls/stake_table.rs +++ b/types/src/v0/impls/stake_table.rs @@ -189,6 +189,8 @@ impl EpochCommittees { .filter(|entry| entry.stake() > U256::zero()) .collect(); + let randomized_committee = generate_stake_cdf(eligible_leaders.clone(), [0u8; 32]); + let committee = Committee { eligible_leaders, stake_table, @@ -200,6 +202,12 @@ impl EpochCommittees { self.state.insert(epoch, committee.clone()); self.state.insert(epoch + 1, committee.clone()); self.state.insert(epoch + 2, committee.clone()); + self.randomized_committees + .insert(epoch, randomized_committee.clone()); + self.randomized_committees + .insert(epoch + 1, randomized_committee.clone()); + self.randomized_committees + .insert(epoch + 1, randomized_committee.clone()); committee } @@ -245,6 +253,7 @@ impl EpochCommittees { .iter() .map(|entry| (PubKey::public_key(entry), entry.clone())) .collect(); + let randomized_committee = generate_stake_cdf(eligible_leaders.clone(), [0u8; 32]); let members = Committee { eligible_leaders, @@ -254,10 +263,13 @@ impl EpochCommittees { indexed_da_members, }; + let mut randomized_committees = BTreeMap::new(); + // TODO: remove this, workaround for hotshot asking for stake tables from epoch 1 and 2 let mut map = HashMap::new(); for epoch in Epoch::genesis().u64()..=50 { map.insert(Epoch::new(epoch), members.clone()); + randomized_committees.insert(Epoch::new(epoch), randomized_committee.clone()); } Self { @@ -267,7 +279,7 @@ impl EpochCommittees { l1_client: instance_state.l1_client.clone(), chain_config: instance_state.chain_config, peers: instance_state.peers.clone(), - randomized_committees: BTreeMap::new(), + randomized_committees, } } From 646b4ad32d78e1c8f3dd8b2c26557959af5a7e7c Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Wed, 26 Feb 2025 14:21:07 +0500 Subject: [PATCH 74/77] move public hotshot config to types crate --- Cargo.lock | 2 + client/Cargo.toml | 1 + client/src/lib.rs | 7 +- sequencer/src/api.rs | 16 +- sequencer/src/api/data_source.rs | 258 +---------------- .../bin/update-permissioned-stake-table.rs | 4 +- sequencer/src/catchup.rs | 7 +- tests/common/mod.rs | 22 +- types/Cargo.toml | 1 + types/src/v0/mod.rs | 3 + types/src/v0/v0_1/config.rs | 262 ++++++++++++++++++ types/src/v0/v0_1/mod.rs | 2 + types/src/v0/v0_2/mod.rs | 9 +- types/src/v0/v0_3/mod.rs | 5 +- types/src/v0/v0_99/mod.rs | 5 +- 15 files changed, 304 insertions(+), 300 deletions(-) create mode 100644 types/src/v0/v0_1/config.rs diff --git a/Cargo.lock b/Cargo.lock index e662a98789..44f4170e75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2497,6 +2497,7 @@ dependencies = [ "futures", "hotshot-types", "jf-merkle-tree", + "serde_json", "surf-disco", "tokio", "tracing", @@ -3754,6 +3755,7 @@ dependencies = [ "tracing", "url", "vbs", + "vec1", ] [[package]] diff --git a/client/Cargo.toml b/client/Cargo.toml index 9fe84cf055..e4d6d51ab5 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -16,3 +16,4 @@ surf-disco = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } vbs = { workspace = true } +serde_json = { workspace = true } \ No newline at end of file diff --git a/client/src/lib.rs b/client/src/lib.rs index f941d96be7..2370201787 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1,5 +1,5 @@ use anyhow::Context; -use espresso_types::{FeeAccount, FeeAmount, FeeMerkleTree, Header, PubKey}; +use espresso_types::{FeeAccount, FeeAmount, FeeMerkleTree, Header, PubKey, PublicNetworkConfig}; use ethers::types::Address; use futures::{stream::BoxStream, StreamExt}; use hotshot_types::stake_table::StakeTableEntry; @@ -10,7 +10,6 @@ use jf_merkle_tree::{ use std::time::Duration; use surf_disco::{ error::ClientError, - http::convert::DeserializeOwned, socket::{Connection, Unsupported}, Url, }; @@ -146,9 +145,9 @@ impl SequencerClient { .context("getting da stake table") } - pub async fn config(&self) -> anyhow::Result { + pub async fn config(&self) -> anyhow::Result { self.0 - .get::("config/hotshot") + .get::("config/hotshot") .send() .await .context("getting hotshot config") diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 4c26e3a660..2c2a4fa246 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -7,8 +7,8 @@ use data_source::{CatchupDataSource, StakeTableDataSource, SubmitDataSource}; use derivative::Derivative; use espresso_types::{ retain_accounts, v0::traits::SequencerPersistence, v0_99::ChainConfig, AccountQueryData, - BlockMerkleTree, FeeAccount, FeeAccountProof, FeeMerkleTree, NodeState, PubKey, Transaction, - ValidatedState, + BlockMerkleTree, FeeAccount, FeeAccountProof, FeeMerkleTree, NodeState, PubKey, + PublicNetworkConfig, Transaction, ValidatedState, }; use futures::{ future::{BoxFuture, Future, FutureExt}, @@ -35,9 +35,7 @@ use jf_merkle_tree::MerkleTreeScheme; use std::pin::Pin; use std::sync::Arc; -use self::data_source::{ - HotShotConfigDataSource, NodeStateDataSource, PublicNetworkConfig, StateSignatureDataSource, -}; +use self::data_source::{HotShotConfigDataSource, NodeStateDataSource, StateSignatureDataSource}; use crate::{ catchup::CatchupStorage, context::Consensus, state_signature::StateSigner, SeqTypes, SequencerApiVersion, SequencerContext, @@ -1612,10 +1610,7 @@ mod test { use tokio::time::sleep; use espresso_types::{ - traits::NullEventConsumer, - v0_1::{UpgradeMode, ViewBasedUpgrade}, - BackoffParams, FeeAccount, FeeAmount, Header, MarketplaceVersion, MockSequencerVersions, - SequencerVersions, TimeBasedUpgrade, Timestamp, Upgrade, UpgradeType, ValidatedState, + traits::NullEventConsumer, v0_1::{UpgradeMode, ViewBasedUpgrade}, BackoffParams, FeeAccount, FeeAmount, Header, MarketplaceVersion, MockSequencerVersions, PublicHotShotConfig, SequencerVersions, TimeBasedUpgrade, Timestamp, Upgrade, UpgradeType, ValidatedState }; use ethers::utils::Anvil; use futures::{ @@ -1645,8 +1640,7 @@ mod test { use vbs::version::{StaticVersion, StaticVersionType, Version}; use self::{ - data_source::{testing::TestableSequencerDataSource, PublicHotShotConfig}, - options::HotshotEvents, + data_source::testing::TestableSequencerDataSource, options::HotshotEvents, sql::DataSource as SqlDataSource, }; use super::*; diff --git a/sequencer/src/api/data_source.rs b/sequencer/src/api/data_source.rs index b8afaa5b95..fc4f8784ab 100644 --- a/sequencer/src/api/data_source.rs +++ b/sequencer/src/api/data_source.rs @@ -1,12 +1,11 @@ -use std::{num::NonZeroUsize, time::Duration}; - use anyhow::Context; use async_trait::async_trait; use committable::Commitment; use espresso_types::{ v0::traits::{PersistenceOptions, SequencerPersistence}, v0_99::ChainConfig, - FeeAccount, FeeAccountProof, FeeMerkleTree, NodeState, PubKey, Transaction, + FeeAccount, FeeAccountProof, FeeMerkleTree, NodeState, PubKey, PublicNetworkConfig, + Transaction, }; use futures::future::Future; use hotshot_query_service::{ @@ -16,21 +15,15 @@ use hotshot_query_service::{ node::NodeDataSource, status::StatusDataSource, }; +use hotshot_types::traits::node_implementation::NodeType; use hotshot_types::{ data::ViewNumber, light_client::StateSignatureRequestBody, - network::NetworkConfig, stake_table::StakeTableEntry, traits::{network::ConnectedNetwork, node_implementation::Versions}, - HotShotConfig, PeerConfig, ValidatorConfig, -}; -use hotshot_types::{ - network::{BuilderType, CombinedNetworkConfig, Libp2pConfig, RandomBuilderConfig}, - traits::node_implementation::NodeType, }; -use serde::{Deserialize, Serialize}; + use tide_disco::Url; -use vec1::Vec1; use super::{ fs, @@ -198,249 +191,6 @@ pub(crate) trait CatchupDataSource: Sync { ) -> impl Send + Future>; } -/// This struct defines the public Hotshot validator configuration. -/// Private key and state key pairs are excluded for security reasons. - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct PublicValidatorConfig { - public_key: PubKey, - stake_value: u64, - is_da: bool, - private_key: String, - state_public_key: String, - state_key_pair: String, -} - -impl From> for PublicValidatorConfig { - fn from(v: ValidatorConfig) -> Self { - let ValidatorConfig:: { - public_key, - private_key: _, - stake_value, - state_key_pair, - is_da, - } = v; - - let state_public_key = state_key_pair.ver_key(); - - Self { - public_key, - stake_value, - is_da, - state_public_key: state_public_key.to_string(), - private_key: "*****".into(), - state_key_pair: "*****".into(), - } - } -} - -/// This struct defines the public Hotshot configuration parameters. -/// Our config module features a GET endpoint accessible via the route `/hotshot` to display the hotshot config parameters. -/// Hotshot config has sensitive information like private keys and such fields are excluded from this struct. -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct PublicHotShotConfig { - start_threshold: (u64, u64), - num_nodes_with_stake: NonZeroUsize, - known_nodes_with_stake: Vec>, - known_da_nodes: Vec>, - da_staked_committee_size: usize, - fixed_leader_for_gpuvid: usize, - next_view_timeout: u64, - view_sync_timeout: Duration, - num_bootstrap: usize, - builder_timeout: Duration, - data_request_delay: Duration, - builder_urls: Vec1, - start_proposing_view: u64, - stop_proposing_view: u64, - start_voting_view: u64, - stop_voting_view: u64, - start_proposing_time: u64, - stop_proposing_time: u64, - start_voting_time: u64, - stop_voting_time: u64, - epoch_height: u64, -} - -impl From> for PublicHotShotConfig { - fn from(v: HotShotConfig) -> Self { - // Destructure all fields from HotShotConfig to return an error - // if new fields are added to HotShotConfig. This makes sure that we handle - // all fields appropriately and do not miss any updates. - let HotShotConfig:: { - start_threshold, - num_nodes_with_stake, - known_nodes_with_stake, - known_da_nodes, - da_staked_committee_size, - fixed_leader_for_gpuvid, - next_view_timeout, - view_sync_timeout, - num_bootstrap, - builder_timeout, - data_request_delay, - builder_urls, - start_proposing_view, - stop_proposing_view, - start_voting_view, - stop_voting_view, - start_proposing_time, - stop_proposing_time, - start_voting_time, - stop_voting_time, - epoch_height, - } = v; - - Self { - start_threshold, - num_nodes_with_stake, - known_nodes_with_stake, - known_da_nodes, - da_staked_committee_size, - fixed_leader_for_gpuvid, - next_view_timeout, - view_sync_timeout, - num_bootstrap, - builder_timeout, - data_request_delay, - builder_urls, - start_proposing_view, - stop_proposing_view, - start_voting_view, - stop_voting_view, - start_proposing_time, - stop_proposing_time, - start_voting_time, - stop_voting_time, - epoch_height, - } - } -} - -impl PublicHotShotConfig { - pub fn into_hotshot_config(self) -> HotShotConfig { - HotShotConfig { - start_threshold: self.start_threshold, - num_nodes_with_stake: self.num_nodes_with_stake, - known_nodes_with_stake: self.known_nodes_with_stake, - known_da_nodes: self.known_da_nodes, - da_staked_committee_size: self.da_staked_committee_size, - fixed_leader_for_gpuvid: self.fixed_leader_for_gpuvid, - next_view_timeout: self.next_view_timeout, - view_sync_timeout: self.view_sync_timeout, - num_bootstrap: self.num_bootstrap, - builder_timeout: self.builder_timeout, - data_request_delay: self.data_request_delay, - builder_urls: self.builder_urls, - start_proposing_view: self.start_proposing_view, - stop_proposing_view: self.stop_proposing_view, - start_voting_view: self.start_voting_view, - stop_voting_view: self.stop_voting_view, - start_proposing_time: self.start_proposing_time, - stop_proposing_time: self.stop_proposing_time, - start_voting_time: self.start_voting_time, - stop_voting_time: self.stop_voting_time, - epoch_height: self.epoch_height, - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct PublicNetworkConfig { - rounds: usize, - indexed_da: bool, - transactions_per_round: usize, - manual_start_password: Option, - num_bootrap: usize, - next_view_timeout: u64, - view_sync_timeout: Duration, - builder_timeout: Duration, - data_request_delay: Duration, - node_index: u64, - seed: [u8; 32], - transaction_size: usize, - key_type_name: String, - libp2p_config: Option, - config: PublicHotShotConfig, - cdn_marshal_address: Option, - combined_network_config: Option, - commit_sha: String, - builder: BuilderType, - random_builder: Option, -} - -impl From> for PublicNetworkConfig { - fn from(cfg: NetworkConfig) -> Self { - Self { - rounds: cfg.rounds, - indexed_da: cfg.indexed_da, - transactions_per_round: cfg.transactions_per_round, - manual_start_password: Some("*****".into()), - num_bootrap: cfg.num_bootrap, - next_view_timeout: cfg.next_view_timeout, - view_sync_timeout: cfg.view_sync_timeout, - builder_timeout: cfg.builder_timeout, - data_request_delay: cfg.data_request_delay, - node_index: cfg.node_index, - seed: cfg.seed, - transaction_size: cfg.transaction_size, - key_type_name: cfg.key_type_name, - libp2p_config: cfg.libp2p_config, - config: cfg.config.into(), - cdn_marshal_address: cfg.cdn_marshal_address, - combined_network_config: cfg.combined_network_config, - commit_sha: cfg.commit_sha, - builder: cfg.builder, - random_builder: cfg.random_builder, - } - } -} - -impl PublicNetworkConfig { - pub fn into_network_config( - self, - my_own_validator_config: ValidatorConfig, - ) -> anyhow::Result> { - let node_index = self - .config - .known_nodes_with_stake - .iter() - .position(|peer| peer.stake_table_entry.stake_key == my_own_validator_config.public_key) - .context(format!( - "the node {} is not in the stake table", - my_own_validator_config.public_key - ))? as u64; - - Ok(NetworkConfig { - rounds: self.rounds, - indexed_da: self.indexed_da, - transactions_per_round: self.transactions_per_round, - manual_start_password: self.manual_start_password, - num_bootrap: self.num_bootrap, - next_view_timeout: self.next_view_timeout, - view_sync_timeout: self.view_sync_timeout, - builder_timeout: self.builder_timeout, - data_request_delay: self.data_request_delay, - node_index, - seed: self.seed, - transaction_size: self.transaction_size, - key_type_name: self.key_type_name, - libp2p_config: self.libp2p_config, - config: self.config.into_hotshot_config(), - cdn_marshal_address: self.cdn_marshal_address, - combined_network_config: self.combined_network_config, - commit_sha: self.commit_sha, - builder: self.builder, - random_builder: self.random_builder, - public_keys: Vec::new(), - }) - } - - pub fn hotshot_config(&self) -> PublicHotShotConfig { - self.config.clone() - } -} - #[cfg(any(test, feature = "testing"))] pub mod testing { use super::{super::Options, *}; diff --git a/sequencer/src/bin/update-permissioned-stake-table.rs b/sequencer/src/bin/update-permissioned-stake-table.rs index d0a3bac1c1..e85cc39d5a 100644 --- a/sequencer/src/bin/update-permissioned-stake-table.rs +++ b/sequencer/src/bin/update-permissioned-stake-table.rs @@ -1,10 +1,10 @@ use anyhow::{Context, Result}; use clap::Parser; use client::SequencerClient; -use espresso_types::parse_duration; +use espresso_types::{parse_duration, PublicNetworkConfig}; use ethers::types::Address; use hotshot_types::{network::PeerConfigKeys, traits::signature_key::StakeTableEntryType}; -use sequencer::api::data_source::PublicNetworkConfig; + use sequencer_utils::{ logging, stake_table::{update_stake_table, PermissionedStakeTableUpdate}, diff --git a/sequencer/src/catchup.rs b/sequencer/src/catchup.rs index 8c8ca7a667..99ea884bda 100644 --- a/sequencer/src/catchup.rs +++ b/sequencer/src/catchup.rs @@ -6,6 +6,7 @@ use async_trait::async_trait; use committable::Commitment; use committable::Committable; use espresso_types::traits::SequencerPersistence; +use espresso_types::PublicNetworkConfig; use espresso_types::{ v0::traits::StateCatchup, v0_99::ChainConfig, BackoffParams, BlockMerkleTree, FeeAccount, FeeAccountProof, FeeMerkleCommitment, FeeMerkleTree, Leaf2, NodeState, @@ -31,10 +32,8 @@ use tokio::time::timeout; use url::Url; use vbs::version::StaticVersionType; -use crate::{ - api::{data_source::PublicNetworkConfig, BlocksFrontier}, - PubKey, -}; +use crate::api::BlocksFrontier; +use crate::PubKey; // This newtype is probably not worth having. It's only used to be able to log // URLs before doing requests. diff --git a/tests/common/mod.rs b/tests/common/mod.rs index f0abe66e0e..dc5f8e989d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,14 +1,13 @@ use anyhow::{anyhow, Context, Result}; use client::SequencerClient; -use espresso_types::{FeeAmount, FeeVersion, MarketplaceVersion, PubKey}; +use espresso_types::{FeeAmount, FeeVersion, MarketplaceVersion}; use ethers::prelude::*; use futures::future::{join_all, BoxFuture}; use futures::FutureExt; use hotshot_types::network::PeerConfigKeys; use hotshot_types::traits::signature_key::StakeTableEntryType; -use hotshot_types::PeerConfig; + use std::{fmt, str::FromStr, time::Duration}; -use surf_disco::http::convert::Deserialize; use surf_disco::Url; use tokio::time::{sleep, timeout}; use vbs::version::StaticVersionType; @@ -364,11 +363,11 @@ pub async fn test_stake_table_update(clients: Vec) -> Result<() .boxed() }; - let config = client.config::().await?.config; + let config = client.config().await?.hotshot_config(); // currently stake table update does not support DA node member changes - let stake_table = config.known_nodes_with_stake; - let da_members = config.known_da_nodes; + let stake_table = config.known_nodes_with_stake(); + let da_members = config.known_da_nodes(); // filtering out DA nodes let non_da_stakers: Vec<_> = stake_table @@ -404,14 +403,3 @@ pub async fn test_stake_table_update(clients: Vec) -> Result<() Ok(()) } - -#[derive(Debug, Deserialize)] -struct PublicHotShotConfig { - known_nodes_with_stake: Vec>, - known_da_nodes: Vec>, -} - -#[derive(Debug, Deserialize)] -struct PublicNetworkConfig { - config: PublicHotShotConfig, -} diff --git a/types/Cargo.toml b/types/Cargo.toml index 7828de6db7..105849e075 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -57,6 +57,7 @@ tower-service = { version = "0.3", default-features = false } tracing = { workspace = true } url = { workspace = true } vbs = { workspace = true } +vec1 = { workspace = true } [dev-dependencies] espresso-types = { path = ".", features = [ "testing" ] } diff --git a/types/src/v0/mod.rs b/types/src/v0/mod.rs index 56bc560582..12440da3d7 100644 --- a/types/src/v0/mod.rs +++ b/types/src/v0/mod.rs @@ -120,6 +120,9 @@ reexport_unchanged_types!( TimeBasedUpgrade, ViewBasedUpgrade, BlockSize, + PublicHotShotConfig, + PublicNetworkConfig, + PublicValidatorConfig ); pub(crate) use v0_3::{L1ClientMetrics, L1Event, L1State, L1UpdateTask}; diff --git a/types/src/v0/v0_1/config.rs b/types/src/v0/v0_1/config.rs new file mode 100644 index 0000000000..4f52574f20 --- /dev/null +++ b/types/src/v0/v0_1/config.rs @@ -0,0 +1,262 @@ +use std::{num::NonZeroUsize, time::Duration}; + +use anyhow::Context; +use vec1::Vec1; + +use crate::PubKey; +use hotshot_types::network::{ + BuilderType, CombinedNetworkConfig, Libp2pConfig, RandomBuilderConfig, +}; +use hotshot_types::{network::NetworkConfig, HotShotConfig, PeerConfig, ValidatorConfig}; +use serde::{Deserialize, Serialize}; +use tide_disco::Url; + +/// This struct defines the public Hotshot validator configuration. +/// Private key and state key pairs are excluded for security reasons. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct PublicValidatorConfig { + public_key: PubKey, + stake_value: u64, + is_da: bool, + private_key: String, + state_public_key: String, + state_key_pair: String, +} + +impl From> for PublicValidatorConfig { + fn from(v: ValidatorConfig) -> Self { + let ValidatorConfig:: { + public_key, + private_key: _, + stake_value, + state_key_pair, + is_da, + } = v; + + let state_public_key = state_key_pair.ver_key(); + + Self { + public_key, + stake_value, + is_da, + state_public_key: state_public_key.to_string(), + private_key: "*****".into(), + state_key_pair: "*****".into(), + } + } +} + +/// This struct defines the public Hotshot configuration parameters. +/// Our config module features a GET endpoint accessible via the route `/hotshot` to display the hotshot config parameters. +/// Hotshot config has sensitive information like private keys and such fields are excluded from this struct. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct PublicHotShotConfig { + start_threshold: (u64, u64), + num_nodes_with_stake: NonZeroUsize, + known_nodes_with_stake: Vec>, + known_da_nodes: Vec>, + da_staked_committee_size: usize, + fixed_leader_for_gpuvid: usize, + next_view_timeout: u64, + view_sync_timeout: Duration, + num_bootstrap: usize, + builder_timeout: Duration, + data_request_delay: Duration, + builder_urls: Vec1, + start_proposing_view: u64, + stop_proposing_view: u64, + start_voting_view: u64, + stop_voting_view: u64, + start_proposing_time: u64, + stop_proposing_time: u64, + start_voting_time: u64, + stop_voting_time: u64, + epoch_height: u64, +} + +impl From> for PublicHotShotConfig { + fn from(v: HotShotConfig) -> Self { + // Destructure all fields from HotShotConfig to return an error + // if new fields are added to HotShotConfig. This makes sure that we handle + // all fields appropriately and do not miss any updates. + let HotShotConfig:: { + start_threshold, + num_nodes_with_stake, + known_nodes_with_stake, + known_da_nodes, + da_staked_committee_size, + fixed_leader_for_gpuvid, + next_view_timeout, + view_sync_timeout, + num_bootstrap, + builder_timeout, + data_request_delay, + builder_urls, + start_proposing_view, + stop_proposing_view, + start_voting_view, + stop_voting_view, + start_proposing_time, + stop_proposing_time, + start_voting_time, + stop_voting_time, + epoch_height, + } = v; + + Self { + start_threshold, + num_nodes_with_stake, + known_nodes_with_stake, + known_da_nodes, + da_staked_committee_size, + fixed_leader_for_gpuvid, + next_view_timeout, + view_sync_timeout, + num_bootstrap, + builder_timeout, + data_request_delay, + builder_urls, + start_proposing_view, + stop_proposing_view, + start_voting_view, + stop_voting_view, + start_proposing_time, + stop_proposing_time, + start_voting_time, + stop_voting_time, + epoch_height, + } + } +} + +impl PublicHotShotConfig { + pub fn into_hotshot_config(self) -> HotShotConfig { + HotShotConfig { + start_threshold: self.start_threshold, + num_nodes_with_stake: self.num_nodes_with_stake, + known_nodes_with_stake: self.known_nodes_with_stake, + known_da_nodes: self.known_da_nodes, + da_staked_committee_size: self.da_staked_committee_size, + fixed_leader_for_gpuvid: self.fixed_leader_for_gpuvid, + next_view_timeout: self.next_view_timeout, + view_sync_timeout: self.view_sync_timeout, + num_bootstrap: self.num_bootstrap, + builder_timeout: self.builder_timeout, + data_request_delay: self.data_request_delay, + builder_urls: self.builder_urls, + start_proposing_view: self.start_proposing_view, + stop_proposing_view: self.stop_proposing_view, + start_voting_view: self.start_voting_view, + stop_voting_view: self.stop_voting_view, + start_proposing_time: self.start_proposing_time, + stop_proposing_time: self.stop_proposing_time, + start_voting_time: self.start_voting_time, + stop_voting_time: self.stop_voting_time, + epoch_height: self.epoch_height, + } + } + + pub fn known_nodes_with_stake(&self) -> Vec> { + self.known_nodes_with_stake.clone() + } + + pub fn known_da_nodes(&self) -> Vec> { + self.known_da_nodes.clone() + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct PublicNetworkConfig { + rounds: usize, + indexed_da: bool, + transactions_per_round: usize, + manual_start_password: Option, + num_bootrap: usize, + next_view_timeout: u64, + view_sync_timeout: Duration, + builder_timeout: Duration, + data_request_delay: Duration, + node_index: u64, + seed: [u8; 32], + transaction_size: usize, + key_type_name: String, + libp2p_config: Option, + config: PublicHotShotConfig, + cdn_marshal_address: Option, + combined_network_config: Option, + commit_sha: String, + builder: BuilderType, + random_builder: Option, +} + +impl From> for PublicNetworkConfig { + fn from(cfg: NetworkConfig) -> Self { + Self { + rounds: cfg.rounds, + indexed_da: cfg.indexed_da, + transactions_per_round: cfg.transactions_per_round, + manual_start_password: Some("*****".into()), + num_bootrap: cfg.num_bootrap, + next_view_timeout: cfg.next_view_timeout, + view_sync_timeout: cfg.view_sync_timeout, + builder_timeout: cfg.builder_timeout, + data_request_delay: cfg.data_request_delay, + node_index: cfg.node_index, + seed: cfg.seed, + transaction_size: cfg.transaction_size, + key_type_name: cfg.key_type_name, + libp2p_config: cfg.libp2p_config, + config: cfg.config.into(), + cdn_marshal_address: cfg.cdn_marshal_address, + combined_network_config: cfg.combined_network_config, + commit_sha: cfg.commit_sha, + builder: cfg.builder, + random_builder: cfg.random_builder, + } + } +} + +impl PublicNetworkConfig { + pub fn into_network_config( + self, + my_own_validator_config: ValidatorConfig, + ) -> anyhow::Result> { + let node_index = self + .config + .known_nodes_with_stake + .iter() + .position(|peer| peer.stake_table_entry.stake_key == my_own_validator_config.public_key) + .context(format!( + "the node {} is not in the stake table", + my_own_validator_config.public_key + ))? as u64; + + Ok(NetworkConfig { + rounds: self.rounds, + indexed_da: self.indexed_da, + transactions_per_round: self.transactions_per_round, + manual_start_password: self.manual_start_password, + num_bootrap: self.num_bootrap, + next_view_timeout: self.next_view_timeout, + view_sync_timeout: self.view_sync_timeout, + builder_timeout: self.builder_timeout, + data_request_delay: self.data_request_delay, + node_index, + seed: self.seed, + transaction_size: self.transaction_size, + key_type_name: self.key_type_name, + libp2p_config: self.libp2p_config, + config: self.config.into_hotshot_config(), + cdn_marshal_address: self.cdn_marshal_address, + combined_network_config: self.combined_network_config, + commit_sha: self.commit_sha, + builder: self.builder, + random_builder: self.random_builder, + public_keys: Vec::new(), + }) + } + + pub fn hotshot_config(&self) -> PublicHotShotConfig { + self.config.clone() + } +} diff --git a/types/src/v0/v0_1/mod.rs b/types/src/v0/v0_1/mod.rs index 7115342c14..494659e1be 100644 --- a/types/src/v0/v0_1/mod.rs +++ b/types/src/v0/v0_1/mod.rs @@ -4,6 +4,7 @@ pub const VERSION: Version = Version { major: 0, minor: 1 }; mod block; mod chain_config; +mod config; mod fee_info; mod header; mod instance_state; @@ -14,6 +15,7 @@ mod transaction; pub use block::*; pub use chain_config::*; +pub use config::*; pub use fee_info::*; pub use header::Header; pub use instance_state::*; diff --git a/types/src/v0/v0_2/mod.rs b/types/src/v0/v0_2/mod.rs index b550f2c5a7..5d8acb79f4 100644 --- a/types/src/v0/v0_2/mod.rs +++ b/types/src/v0/v0_2/mod.rs @@ -4,10 +4,11 @@ use vbs::version::Version; pub use super::v0_1::{ AccountQueryData, BlockMerkleCommitment, BlockMerkleTree, BlockSize, BuilderSignature, ChainConfig, ChainId, Delta, FeeAccount, FeeAccountProof, FeeAmount, FeeInfo, - FeeMerkleCommitment, FeeMerkleProof, FeeMerkleTree, Header, Index, Iter, L1BlockInfo, L1Client, L1ClientOptions, - L1Snapshot, NamespaceId, NsIndex, NsIter, NsPayload, NsPayloadBuilder, NsPayloadByteLen, - NsPayloadOwned, NsPayloadRange, NsProof, NsTable, NsTableBuilder, NsTableValidationError, - NumNss, NumTxs, NumTxsRange, NumTxsUnchecked, Payload, PayloadByteLen, ResolvableChainConfig, + FeeMerkleCommitment, FeeMerkleProof, FeeMerkleTree, Header, Index, Iter, L1BlockInfo, L1Client, + L1ClientOptions, L1Snapshot, NamespaceId, NsIndex, NsIter, NsPayload, NsPayloadBuilder, + NsPayloadByteLen, NsPayloadOwned, NsPayloadRange, NsProof, NsTable, NsTableBuilder, + NsTableValidationError, NumNss, NumTxs, NumTxsRange, NumTxsUnchecked, Payload, PayloadByteLen, + PublicHotShotConfig, PublicNetworkConfig, PublicValidatorConfig, ResolvableChainConfig, TimeBasedUpgrade, Transaction, TxIndex, TxIter, TxPayload, TxPayloadRange, TxProof, TxTableEntries, TxTableEntriesRange, Upgrade, UpgradeMode, UpgradeType, ViewBasedUpgrade, BLOCK_MERKLE_TREE_HEIGHT, FEE_MERKLE_TREE_HEIGHT, NS_ID_BYTE_LEN, NS_OFFSET_BYTE_LEN, diff --git a/types/src/v0/v0_3/mod.rs b/types/src/v0/v0_3/mod.rs index ec59b858b4..b96fb014a9 100644 --- a/types/src/v0/v0_3/mod.rs +++ b/types/src/v0/v0_3/mod.rs @@ -7,8 +7,9 @@ pub use super::v0_1::{ FeeMerkleTree, Index, Iter, L1BlockInfo, L1Client, L1ClientOptions, L1Snapshot, NamespaceId, NsIndex, NsIter, NsPayload, NsPayloadBuilder, NsPayloadByteLen, NsPayloadOwned, NsPayloadRange, NsProof, NsTable, NsTableBuilder, NsTableValidationError, NumNss, NumTxs, NumTxsRange, - NumTxsUnchecked, Payload, PayloadByteLen, TimeBasedUpgrade, Transaction, TxIndex, TxIter, - TxPayload, TxPayloadRange, TxProof, TxTableEntries, TxTableEntriesRange, Upgrade, UpgradeMode, + NumTxsUnchecked, Payload, PayloadByteLen, PublicHotShotConfig, PublicNetworkConfig, + PublicValidatorConfig, TimeBasedUpgrade, Transaction, TxIndex, TxIter, TxPayload, + TxPayloadRange, TxProof, TxTableEntries, TxTableEntriesRange, Upgrade, UpgradeMode, UpgradeType, ViewBasedUpgrade, BLOCK_MERKLE_TREE_HEIGHT, FEE_MERKLE_TREE_HEIGHT, NS_ID_BYTE_LEN, NS_OFFSET_BYTE_LEN, NUM_NSS_BYTE_LEN, NUM_TXS_BYTE_LEN, TX_OFFSET_BYTE_LEN, }; diff --git a/types/src/v0/v0_99/mod.rs b/types/src/v0/v0_99/mod.rs index 3e676a4b2a..f45ce0bd5d 100644 --- a/types/src/v0/v0_99/mod.rs +++ b/types/src/v0/v0_99/mod.rs @@ -7,8 +7,9 @@ pub use super::v0_1::{ FeeMerkleTree, Index, Iter, L1BlockInfo, L1Client, L1ClientOptions, L1Snapshot, NamespaceId, NsIndex, NsIter, NsPayload, NsPayloadBuilder, NsPayloadByteLen, NsPayloadOwned, NsPayloadRange, NsProof, NsTable, NsTableBuilder, NsTableValidationError, NumNss, NumTxs, NumTxsRange, - NumTxsUnchecked, Payload, PayloadByteLen, TimeBasedUpgrade, Transaction, TxIndex, TxIter, - TxPayload, TxPayloadRange, TxProof, TxTableEntries, TxTableEntriesRange, Upgrade, UpgradeMode, + NumTxsUnchecked, Payload, PayloadByteLen, PublicHotShotConfig, PublicNetworkConfig, + PublicValidatorConfig, TimeBasedUpgrade, Transaction, TxIndex, TxIter, TxPayload, + TxPayloadRange, TxProof, TxTableEntries, TxTableEntriesRange, Upgrade, UpgradeMode, UpgradeType, ViewBasedUpgrade, BLOCK_MERKLE_TREE_HEIGHT, FEE_MERKLE_TREE_HEIGHT, NS_ID_BYTE_LEN, NS_OFFSET_BYTE_LEN, NUM_NSS_BYTE_LEN, NUM_TXS_BYTE_LEN, TX_OFFSET_BYTE_LEN, }; From e18cb5a4da72004632b0746162b81f5bd2dabaad Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Wed, 26 Feb 2025 14:31:44 +0500 Subject: [PATCH 75/77] fix randomized committee preloading --- sequencer-sqlite/Cargo.lock | 2 ++ sequencer/src/api.rs | 6 +++++- sequencer/src/bin/update-permissioned-stake-table.rs | 4 ++-- types/src/v0/impls/stake_table.rs | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sequencer-sqlite/Cargo.lock b/sequencer-sqlite/Cargo.lock index e4b942b022..62b192f64a 100644 --- a/sequencer-sqlite/Cargo.lock +++ b/sequencer-sqlite/Cargo.lock @@ -2388,6 +2388,7 @@ dependencies = [ "futures", "hotshot-types", "jf-merkle-tree", + "serde_json", "surf-disco", "tokio", "tracing", @@ -3579,6 +3580,7 @@ dependencies = [ "tracing", "url", "vbs", + "vec1", ] [[package]] diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 2c2a4fa246..d637767621 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -1610,7 +1610,11 @@ mod test { use tokio::time::sleep; use espresso_types::{ - traits::NullEventConsumer, v0_1::{UpgradeMode, ViewBasedUpgrade}, BackoffParams, FeeAccount, FeeAmount, Header, MarketplaceVersion, MockSequencerVersions, PublicHotShotConfig, SequencerVersions, TimeBasedUpgrade, Timestamp, Upgrade, UpgradeType, ValidatedState + traits::NullEventConsumer, + v0_1::{UpgradeMode, ViewBasedUpgrade}, + BackoffParams, FeeAccount, FeeAmount, Header, MarketplaceVersion, MockSequencerVersions, + PublicHotShotConfig, SequencerVersions, TimeBasedUpgrade, Timestamp, Upgrade, UpgradeType, + ValidatedState, }; use ethers::utils::Anvil; use futures::{ diff --git a/sequencer/src/bin/update-permissioned-stake-table.rs b/sequencer/src/bin/update-permissioned-stake-table.rs index e85cc39d5a..a966a30f22 100644 --- a/sequencer/src/bin/update-permissioned-stake-table.rs +++ b/sequencer/src/bin/update-permissioned-stake-table.rs @@ -1,7 +1,7 @@ use anyhow::{Context, Result}; use clap::Parser; use client::SequencerClient; -use espresso_types::{parse_duration, PublicNetworkConfig}; +use espresso_types::parse_duration; use ethers::types::Address; use hotshot_types::{network::PeerConfigKeys, traits::signature_key::StakeTableEntryType}; @@ -109,7 +109,7 @@ async fn main() -> Result<()> { for client in &clients { tracing::warn!("calling config endpoint of {client:?}"); - match client.config::().await { + match client.config().await { Ok(config) => { let hotshot = config.hotshot_config().into_hotshot_config(); let st = hotshot.known_nodes_with_stake; diff --git a/types/src/v0/impls/stake_table.rs b/types/src/v0/impls/stake_table.rs index 49d164badb..da2a90662b 100644 --- a/types/src/v0/impls/stake_table.rs +++ b/types/src/v0/impls/stake_table.rs @@ -207,7 +207,7 @@ impl EpochCommittees { self.randomized_committees .insert(epoch + 1, randomized_committee.clone()); self.randomized_committees - .insert(epoch + 1, randomized_committee.clone()); + .insert(epoch + 2, randomized_committee.clone()); committee } From 9bc5e575e45e155eda1deeadc72856e4c6d059f4 Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Wed, 26 Feb 2025 17:10:38 +0500 Subject: [PATCH 76/77] assert upgrade 10 views after upgrade proposal --- sequencer/src/api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index d637767621..c76207e192 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -2357,7 +2357,7 @@ mod test { // ChainConfigs will eventually be resolved if let Some(configs) = configs { tracing::info!(?configs, "configs"); - if height > new_version_first_view { + if height > new_version_first_view + 10 { for config in configs { assert_eq!(config, chain_config_upgrade); } From 6f0790824558504358cacbfe1aa534ce8806e03f Mon Sep 17 00:00:00 2001 From: imabdulbasit Date: Thu, 27 Feb 2025 03:13:12 +0500 Subject: [PATCH 77/77] fix: off by one epoch for getting drb results --- client/Cargo.toml | 2 +- hotshot-task-impls/src/quorum_proposal/handlers.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/Cargo.toml b/client/Cargo.toml index e4d6d51ab5..fb65f0ce4b 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -12,8 +12,8 @@ ethers = { workspace = true } futures = { workspace = true } hotshot-types = { workspace = true } jf-merkle-tree = { workspace = true } +serde_json = { workspace = true } surf-disco = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } vbs = { workspace = true } -serde_json = { workspace = true } \ No newline at end of file diff --git a/hotshot-task-impls/src/quorum_proposal/handlers.rs b/hotshot-task-impls/src/quorum_proposal/handlers.rs index 6e4e73d671..22c24c85e5 100644 --- a/hotshot-task-impls/src/quorum_proposal/handlers.rs +++ b/hotshot-task-impls/src/quorum_proposal/handlers.rs @@ -407,7 +407,7 @@ impl ProposalDependencyHandle { .await .drb_seeds_and_results .results - .get(epoch_val) + .get(&(*epoch_val + 1)) .copied() } else { None