From 196c1fb6622d9ef7599a462ae5f73858cce198dd Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Thu, 6 Feb 2025 18:41:51 +0200 Subject: [PATCH 01/35] feat!: Celestia mainnet config Redo the celestia DA configuration: transpose the network enum and the parameter struct, so that network is a field. This makes for a lot less code duplication. --- .../migrations/elsa_to_biarritz_rc1/mod.rs | 7 +- .../src/node/tasks/transaction_ingress.rs | 4 +- .../src/celestia_appd/local.rs | 4 +- .../celestia-runners/src/celestia_appd/mod.rs | 16 +- .../src/celestia_bridge/local.rs | 7 +- .../src/celestia_bridge/mod.rs | 14 +- .../src/celestia_light/arabica.rs | 2 +- .../src/celestia_light/mainnet.rs | 32 ++ .../src/celestia_light/mocha.rs | 2 +- .../src/celestia_light/mod.rs | 17 +- .../protocol/light-node/src/sequencer.rs | 2 +- .../da/movement/protocol/setup/src/arabica.rs | 2 +- .../protocol/setup/src/common/celestia.rs | 7 +- .../protocol/setup/src/common/memseq.rs | 2 +- .../da/movement/protocol/setup/src/lib.rs | 25 +- .../da/movement/protocol/setup/src/local.rs | 2 +- .../da/movement/protocol/setup/src/mainnet.rs | 128 ++++++++ .../da/movement/protocol/setup/src/mocha.rs | 2 +- .../util/src/config/{local => }/appd.rs | 16 +- .../util/src/config/{local => }/bridge.rs | 2 +- .../src/config/{local => }/da_light_node.rs | 2 +- .../util/src/config/{common.rs => default.rs} | 0 .../src/config/{local => }/digest_store.rs | 0 .../protocol/util/src/config/local/mod.rs | 57 ---- .../movement/protocol/util/src/config/mod.rs | 308 ++++++------------ 25 files changed, 346 insertions(+), 314 deletions(-) create mode 100644 protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs create mode 100644 protocol-units/da/movement/protocol/setup/src/mainnet.rs rename protocol-units/da/movement/protocol/util/src/config/{local => }/appd.rs (86%) rename protocol-units/da/movement/protocol/util/src/config/{local => }/bridge.rs (98%) rename protocol-units/da/movement/protocol/util/src/config/{local => }/da_light_node.rs (99%) rename protocol-units/da/movement/protocol/util/src/config/{common.rs => default.rs} (100%) rename protocol-units/da/movement/protocol/util/src/config/{local => }/digest_store.rs (100%) delete mode 100644 protocol-units/da/movement/protocol/util/src/config/local/mod.rs diff --git a/networks/movement/movement-config/src/migrations/elsa_to_biarritz_rc1/mod.rs b/networks/movement/movement-config/src/migrations/elsa_to_biarritz_rc1/mod.rs index a8be7a85a..a52eeaefa 100644 --- a/networks/movement/movement-config/src/migrations/elsa_to_biarritz_rc1/mod.rs +++ b/networks/movement/movement-config/src/migrations/elsa_to_biarritz_rc1/mod.rs @@ -607,8 +607,11 @@ mod tests { }) ); - let da_signers = match &config.celestia_da_light_node.celestia_da_light_node_config { - movement_da_util::config::Config::Local(local) => &local.da_light_node.da_signers, + let da_light_node_config = &config.celestia_da_light_node.celestia_da_light_node_config; + let da_signers = match &da_light_node_config.network { + movement_da_util::config::Network::Local => { + &da_light_node_config.da_light_node.da_signers + } _ => panic!("Expected Local"), }; diff --git a/networks/movement/movement-full-node/src/node/tasks/transaction_ingress.rs b/networks/movement/movement-full-node/src/node/tasks/transaction_ingress.rs index 83383d138..150184d5d 100644 --- a/networks/movement/movement-full-node/src/node/tasks/transaction_ingress.rs +++ b/networks/movement/movement-full-node/src/node/tasks/transaction_ingress.rs @@ -1,9 +1,9 @@ //! Task to process incoming transactions and write to DA use maptos_dof_execution::SignedTransaction; -use movement_da_util::config::Config as LightNodeConfig; use movement_da_light_node_client::MovementDaLightNodeClient; use movement_da_light_node_proto::{BatchWriteRequest, BlobWrite}; +use movement_da_util::config::Config as LightNodeConfig; use tokio::sync::mpsc; use tracing::{info, warn}; @@ -43,7 +43,7 @@ impl Task { // limit the total time batching transactions let start = Instant::now(); - let (_, half_building_time) = self.da_light_node_config.try_block_building_parameters()?; + let (_, half_building_time) = self.da_light_node_config.block_building_parameters(); let mut transactions = Vec::new(); diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/local.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/local.rs index 5422d2719..5bc375b5f 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/local.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/local.rs @@ -10,8 +10,8 @@ impl Local { pub async fn run( &self, - dot_movement: dot_movement::DotMovement, - config: movement_da_util::config::local::Config, + _dot_movement: dot_movement::DotMovement, + config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { // celestia-appd start --grpc.enable --home $CELESTIA_APP_PATH --log_level $LOG_LEVEL diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/mod.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/mod.rs index a76530496..63a43caee 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/mod.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/mod.rs @@ -1,6 +1,6 @@ pub mod local; use crate::Runner; -use movement_da_util::config::CelestiaDaLightNodeConfig; +use movement_da_util::config::{CelestiaDaLightNodeConfig, Network}; #[derive(Debug, Clone)] pub struct CelestiaAppd {} @@ -11,18 +11,16 @@ impl Runner for CelestiaAppd { dot_movement: dot_movement::DotMovement, config: CelestiaDaLightNodeConfig, ) -> Result<(), anyhow::Error> { - match config.celestia_da_light_node_config { - movement_da_util::config::Config::Local(config) => { + let config = config.celestia_da_light_node_config; + match config.network { + Network::Local => { let local = local::Local::new(); local.run(dot_movement, config).await?; Ok(()) } - movement_da_util::config::Config::Arabica(config) => { - Err(anyhow::anyhow!("Arabica not implemented")) - } - movement_da_util::config::Config::Mocha(config) => { - Err(anyhow::anyhow!("Mocha not implemented")) - } + Network::Arabica => Err(anyhow::anyhow!("Arabica not implemented")), + Network::Mocha => Err(anyhow::anyhow!("Mocha not implemented")), + Network::Mainnet => Err(anyhow::anyhow!("Mainnet not implemented")), } } } diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/local.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/local.rs index f846b0840..9bf4cdf1b 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/local.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/local.rs @@ -13,10 +13,7 @@ impl Local { Local } - async fn get_genesis_block( - &self, - config: &movement_da_util::config::local::Config, - ) -> Result { + async fn get_genesis_block(&self, config: &movement_da_util::config::Config) -> Result { let client = Client::new(); let mut genesis = String::new(); let mut cnt = 0; @@ -59,7 +56,7 @@ impl Local { pub async fn run( &self, _dot_movement: dot_movement::DotMovement, - config: movement_da_util::config::local::Config, + config: movement_da_util::config::Config, ) -> Result<()> { let genesis = self.get_genesis_block(&config).await?; diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/mod.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/mod.rs index b3c2fcd4a..0dfb002fb 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/mod.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_bridge/mod.rs @@ -1,5 +1,5 @@ pub mod local; -use movement_da_util::config::CelestiaDaLightNodeConfig; +use movement_da_util::config::{CelestiaDaLightNodeConfig, Network}; use crate::Runner; @@ -12,17 +12,21 @@ impl Runner for CelestiaBridge { dot_movement: dot_movement::DotMovement, config: CelestiaDaLightNodeConfig, ) -> Result<(), anyhow::Error> { - match config.celestia_da_light_node_config { - movement_da_util::config::Config::Local(config) => { + let config = config.celestia_da_light_node_config; + match config.network { + Network::Local => { let local = local::Local::new(); local.run(dot_movement, config).await?; } - movement_da_util::config::Config::Arabica(_config) => { + Network::Arabica => { Err(anyhow::anyhow!("Arabica not implemented"))?; } - movement_da_util::config::Config::Mocha(_config) => { + Network::Mocha => { Err(anyhow::anyhow!("Mocha not implemented"))?; } + Network::Mainnet => { + Err(anyhow::anyhow!("Mainnet not implemented"))?; + } } Ok(()) } diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs index b6c897486..648b36e35 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs @@ -9,7 +9,7 @@ impl Arabica { pub async fn run( &self, _dot_movement: dot_movement::DotMovement, - _config: movement_da_util::config::local::Config, + _config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { // celestia light start --core.ip validator-1.celestia-arabica-11.com --p2p.network arabica commander::run_command( diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs new file mode 100644 index 000000000..a07addc3e --- /dev/null +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs @@ -0,0 +1,32 @@ +#[derive(Debug, Clone)] +pub struct Mainnet; + +impl Mainnet { + pub fn new() -> Self { + Mainnet + } + + pub async fn run( + &self, + _dot_movement: dot_movement::DotMovement, + _config: movement_da_util::config::Config, + ) -> Result<(), anyhow::Error> { + // celestia light start --core.ip ??? --p2p.network celestia + commander::run_command( + "celestia", + &[ + "light", + "start", + "--core.ip", + "rpc.celestia.pops.one", // FIXME: is this correct? + "--p2p.network", + "celestia", + "--log.level", + "FATAL", + ], + ) + .await?; + + Ok(()) + } +} diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs index 531320d8d..30bd0ada3 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs @@ -9,7 +9,7 @@ impl Mocha { pub async fn run( &self, _dot_movement: dot_movement::DotMovement, - _config: movement_da_util::config::local::Config, + _config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { // celestia light start --core.ip validator-1.celestia-mocha-11.com --p2p.network mocha commander::run_command( diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mod.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mod.rs index c9bb97ce4..fef98bf1c 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mod.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mod.rs @@ -1,6 +1,8 @@ pub mod arabica; +pub mod mainnet; pub mod mocha; -use movement_da_util::config::CelestiaDaLightNodeConfig; + +use movement_da_util::config::{CelestiaDaLightNodeConfig, Network}; use crate::Runner; @@ -13,18 +15,23 @@ impl Runner for CelestiaLight { dot_movement: dot_movement::DotMovement, config: CelestiaDaLightNodeConfig, ) -> Result<(), anyhow::Error> { - match config.celestia_da_light_node_config { - movement_da_util::config::Config::Local(_config) => { + let config = config.celestia_da_light_node_config; + match config.network { + Network::Local => { Err(anyhow::anyhow!("Local not implemented"))?; } - movement_da_util::config::Config::Arabica(config) => { + Network::Arabica => { let arabica = arabica::Arabica::new(); arabica.run(dot_movement, config).await?; } - movement_da_util::config::Config::Mocha(config) => { + Network::Mocha => { let mocha = mocha::Mocha::new(); mocha.run(dot_movement, config).await?; } + Network::Mainnet => { + let mainnet = mainnet::Mainnet::new(); + mainnet.run(dot_movement, config).await?; + } } Ok(()) } diff --git a/protocol-units/da/movement/protocol/light-node/src/sequencer.rs b/protocol-units/da/movement/protocol/light-node/src/sequencer.rs index 044ff1f8b..cfc8876e5 100644 --- a/protocol-units/da/movement/protocol/light-node/src/sequencer.rs +++ b/protocol-units/da/movement/protocol/light-node/src/sequencer.rs @@ -91,7 +91,7 @@ impl LightNodeRuntime let memseq_path = pass_through.config.try_memseq_path()?; info!("Memseq path: {:?}", memseq_path); - let (max_block_size, build_time) = pass_through.config.try_block_building_parameters()?; + let (max_block_size, build_time) = pass_through.config.block_building_parameters(); let memseq = Arc::new(memseq::Memseq::try_move_rocks( PathBuf::from(memseq_path), diff --git a/protocol-units/da/movement/protocol/setup/src/arabica.rs b/protocol-units/da/movement/protocol/setup/src/arabica.rs index 56cf410e0..5b54f45e4 100644 --- a/protocol-units/da/movement/protocol/setup/src/arabica.rs +++ b/protocol-units/da/movement/protocol/setup/src/arabica.rs @@ -2,7 +2,7 @@ use crate::common; use anyhow::Context; use commander::run_command; use dot_movement::DotMovement; -use movement_da_util::config::local::Config; +use movement_da_util::config::Config; use tracing::info; #[derive(Debug, Clone)] diff --git a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs index 47dada518..600f97fdd 100644 --- a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs +++ b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs @@ -2,7 +2,7 @@ use crate::common; use anyhow::Context; use celestia_types::nmt::Namespace; use dot_movement::DotMovement; -use movement_da_util::config::local::Config; +use movement_da_util::config::Config; use rand::Rng; use tracing::info; @@ -64,7 +64,10 @@ pub fn initialize_celestia_config( Ok(config) } -pub async fn make_dirs(dot_movement: DotMovement, config: Config) -> Result { +pub async fn make_dirs( + _dot_movement: DotMovement, + config: Config, +) -> Result { // make the celestia app directory let app_path = config.appd.celestia_path.clone().context( "Failed to get Celestia App path from config. This is required for creating the Celestia App directory.", diff --git a/protocol-units/da/movement/protocol/setup/src/common/memseq.rs b/protocol-units/da/movement/protocol/setup/src/common/memseq.rs index 1016eacab..983668b2c 100644 --- a/protocol-units/da/movement/protocol/setup/src/common/memseq.rs +++ b/protocol-units/da/movement/protocol/setup/src/common/memseq.rs @@ -1,5 +1,5 @@ use dot_movement::DotMovement; -use movement_da_util::config::local::Config; +use movement_da_util::config::Config; pub fn initialize_memseq_config( dot_movement: DotMovement, diff --git a/protocol-units/da/movement/protocol/setup/src/lib.rs b/protocol-units/da/movement/protocol/setup/src/lib.rs index 637c6e4d8..cda61e855 100644 --- a/protocol-units/da/movement/protocol/setup/src/lib.rs +++ b/protocol-units/da/movement/protocol/setup/src/lib.rs @@ -1,28 +1,31 @@ pub mod arabica; pub mod common; pub mod local; +pub mod mainnet; pub mod mocha; -use movement_da_util::config::CelestiaDaLightNodeConfig; +use movement_da_util::config::{CelestiaDaLightNodeConfig, Network}; pub async fn setup( dot_movement: dot_movement::DotMovement, mut config: CelestiaDaLightNodeConfig, ) -> Result { - let inner_config = match config.celestia_da_light_node_config { - movement_da_util::config::Config::Local(config) => { + let inner_config = config.celestia_da_light_node_config; + let inner_config = match inner_config.network { + Network::Local => { let local = local::Local::new(); - let local_config = local.setup(dot_movement, config).await?; - movement_da_util::config::Config::Local(local_config) + local.setup(dot_movement, inner_config).await? } - movement_da_util::config::Config::Arabica(config) => { + Network::Arabica => { let arabica = arabica::Arabica::new(); - let arabica_config = arabica.setup(dot_movement, config).await?; - movement_da_util::config::Config::Arabica(arabica_config) + arabica.setup(dot_movement, inner_config).await? } - movement_da_util::config::Config::Mocha(config) => { + Network::Mocha => { let mocha = mocha::Mocha::new(); - let mocha_config = mocha.setup(dot_movement, config).await?; - movement_da_util::config::Config::Mocha(mocha_config) + mocha.setup(dot_movement, inner_config).await? + } + Network::Mainnet => { + let mainnet = mainnet::Mainnet::new(); + mainnet.setup(dot_movement, inner_config).await? } }; config.celestia_da_light_node_config = inner_config; diff --git a/protocol-units/da/movement/protocol/setup/src/local.rs b/protocol-units/da/movement/protocol/setup/src/local.rs index 9e6310641..2f267e950 100644 --- a/protocol-units/da/movement/protocol/setup/src/local.rs +++ b/protocol-units/da/movement/protocol/setup/src/local.rs @@ -2,7 +2,7 @@ use crate::common; use anyhow::Context; use commander::run_command; use dot_movement::DotMovement; -use movement_da_util::config::local::Config; +use movement_da_util::config::Config; use tokio::fs; use tracing::info; diff --git a/protocol-units/da/movement/protocol/setup/src/mainnet.rs b/protocol-units/da/movement/protocol/setup/src/mainnet.rs new file mode 100644 index 000000000..640fdddec --- /dev/null +++ b/protocol-units/da/movement/protocol/setup/src/mainnet.rs @@ -0,0 +1,128 @@ +use crate::common; +use anyhow::Context; +use commander::run_command; +use dot_movement::DotMovement; +use movement_da_util::config::Config; +use tracing::info; + +#[derive(Debug, Clone)] +pub struct Mainnet; + +impl Mainnet { + pub fn new() -> Self { + Self + } + + pub async fn get_mainnet_11_address(&self) -> Result { + // get the json from celkey + // cel-key list --node.type light --keyring-backend test --p2p.network celestia --output json + let json_string = run_command( + "cel-key", + &[ + "list", + "--node.type", + "light", + "--keyring-backend", + "test", + "--p2p.network", + "celestia", + "--output", + "json", + ], + ) + .await?; + + let json_string = json_string + .lines() + .last() + .context("Failed to get the last line of the json string.")?; + + info!("Mainnet 11 address json: {}", json_string); + + // use serde to convert to json + let json: serde_json::Value = serde_json::from_str(&json_string) + .context("Failed to convert json string to json value for celestia address.")?; + + // q -r '.[0].address' + let address = json + .get(0) + .context("Failed to get the first element of the json array.")? + .get("address") + .context("Failed to get the address field from the json object.")? + .as_str() + .context("Failed to convert the address field to a string.")?; + + Ok(address.to_string()) + } + + pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { + // celestia light init --p2p.network celestia + run_command("celestia", &["light", "init", "--p2p.network", "celestia"]).await?; + + Ok(()) + } + + pub async fn get_da_block_height(&self) -> Result { + todo!("what's this for mainnet?"); + let response = reqwest::get("https://rpc-???.pops.one/block").await?.text().await?; + + Ok(response.parse().context("Failed to parse the response to a u64.")?) + } + + pub async fn get_auth_token(&self) -> Result { + // celestia light auth admin --p2p.network celestia + let auth_token = + run_command("celestia", &["light", "auth", "admin", "--p2p.network", "celestia"]) + .await? + .trim() + .to_string(); + + Ok(auth_token) + } + + pub async fn setup_celestia( + &self, + dot_movement: DotMovement, + config: Config, + ) -> Result { + let config = common::celestia::initialize_celestia_config(dot_movement.clone(), config)?; + let config = common::memseq::initialize_memseq_config(dot_movement.clone(), config)?; + let mut config = common::celestia::make_dirs(dot_movement.clone(), config).await?; + + // celestia light init --p2p.network celestia + self.celestia_light_init().await?; + + // get the mainnet 11 address + let address = self.get_mainnet_11_address().await?; + config.appd.celestia_validator_address.replace(address.clone()); + + // get the auth token + let auth_token = self.get_auth_token().await?; + config.appd.celestia_auth_token.replace(auth_token.clone()); + + Ok(config) + } + + pub async fn setup( + &self, + dot_movement: DotMovement, + config: Config, + ) -> Result { + // By default the M1 DA Light Node is not initialized. + if !config.da_light_node_is_initial { + info!("M1 DA Light Node is already initialized."); + return Ok(config); + } + + info!("Setting up Celestia for M1 DA Light Node."); + let mut config = self.setup_celestia(dot_movement, config).await?; + + info!("M1 DA Light Node setup complete."); + + // Now we set the config to initialized. + config.da_light_node_is_initial = false; + + // Placeholder for returning the actual configuration. + Ok(config) + } +} diff --git a/protocol-units/da/movement/protocol/setup/src/mocha.rs b/protocol-units/da/movement/protocol/setup/src/mocha.rs index 3e5e86b60..7b67f3847 100644 --- a/protocol-units/da/movement/protocol/setup/src/mocha.rs +++ b/protocol-units/da/movement/protocol/setup/src/mocha.rs @@ -2,7 +2,7 @@ use crate::common; use anyhow::Context; use commander::run_command; use dot_movement::DotMovement; -use movement_da_util::config::local::Config; +use movement_da_util::config::Config; use tracing::info; #[derive(Debug, Clone)] diff --git a/protocol-units/da/movement/protocol/util/src/config/local/appd.rs b/protocol-units/da/movement/protocol/util/src/config/appd.rs similarity index 86% rename from protocol-units/da/movement/protocol/util/src/config/local/appd.rs rename to protocol-units/da/movement/protocol/util/src/config/appd.rs index 2b61d5983..1be2ab9ae 100644 --- a/protocol-units/da/movement/protocol/util/src/config/local/appd.rs +++ b/protocol-units/da/movement/protocol/util/src/config/appd.rs @@ -1,4 +1,4 @@ -use crate::config::common::{ +use crate::config::default::{ default_celestia_appd_replace_args, default_celestia_appd_use_replace_args, default_celestia_chain_id, default_celestia_namespace, default_celestia_rpc_listen_hostname, default_celestia_rpc_listen_port, default_celestia_websocket_connection_hostname, @@ -79,3 +79,17 @@ impl Default for Config { } } } + +impl Config { + // FIXME: use a single URL field as the source. The format was introduced by + // Sir Tim Berners-Lee in 1994 so you don't have to compose this from + // values that need to be set in three different environment variables. + pub fn celestia_websocket_url(&self) -> String { + format!( + "{}://{}:{}", + self.celestia_websocket_connection_protocol, + self.celestia_websocket_connection_hostname, + self.celestia_websocket_connection_port + ) + } +} diff --git a/protocol-units/da/movement/protocol/util/src/config/local/bridge.rs b/protocol-units/da/movement/protocol/util/src/config/bridge.rs similarity index 98% rename from protocol-units/da/movement/protocol/util/src/config/local/bridge.rs rename to protocol-units/da/movement/protocol/util/src/config/bridge.rs index 4f5a1bd39..c159d39c4 100644 --- a/protocol-units/da/movement/protocol/util/src/config/local/bridge.rs +++ b/protocol-units/da/movement/protocol/util/src/config/bridge.rs @@ -1,4 +1,4 @@ -use crate::config::common::{ +use crate::config::default::{ default_celestia_bridge_replace_args, default_celestia_bridge_use_replace_args, default_celestia_rpc_connection_hostname, default_celestia_rpc_connection_port, default_celestia_rpc_connection_protocol, default_celestia_websocket_listen_hostname, diff --git a/protocol-units/da/movement/protocol/util/src/config/local/da_light_node.rs b/protocol-units/da/movement/protocol/util/src/config/da_light_node.rs similarity index 99% rename from protocol-units/da/movement/protocol/util/src/config/local/da_light_node.rs rename to protocol-units/da/movement/protocol/util/src/config/da_light_node.rs index 0699daf1f..6009b1994 100644 --- a/protocol-units/da/movement/protocol/util/src/config/local/da_light_node.rs +++ b/protocol-units/da/movement/protocol/util/src/config/da_light_node.rs @@ -1,4 +1,4 @@ -use crate::config::common::{ +use crate::config::default::{ default_celestia_rpc_connection_hostname, default_celestia_rpc_connection_port, default_celestia_rpc_connection_protocol, default_celestia_websocket_connection_hostname, default_celestia_websocket_connection_port, default_movement_da_light_node_connection_hostname, diff --git a/protocol-units/da/movement/protocol/util/src/config/common.rs b/protocol-units/da/movement/protocol/util/src/config/default.rs similarity index 100% rename from protocol-units/da/movement/protocol/util/src/config/common.rs rename to protocol-units/da/movement/protocol/util/src/config/default.rs diff --git a/protocol-units/da/movement/protocol/util/src/config/local/digest_store.rs b/protocol-units/da/movement/protocol/util/src/config/digest_store.rs similarity index 100% rename from protocol-units/da/movement/protocol/util/src/config/local/digest_store.rs rename to protocol-units/da/movement/protocol/util/src/config/digest_store.rs diff --git a/protocol-units/da/movement/protocol/util/src/config/local/mod.rs b/protocol-units/da/movement/protocol/util/src/config/local/mod.rs deleted file mode 100644 index 6b09006ab..000000000 --- a/protocol-units/da/movement/protocol/util/src/config/local/mod.rs +++ /dev/null @@ -1,57 +0,0 @@ -pub mod appd; -pub mod bridge; -pub mod da_light_node; -pub mod digest_store; -use crate::config::common::{default_celestia_force_new_chain, default_da_light_node_is_initial}; -use aptos_account_whitelist::config::Config as WhitelistConfig; -use memseq_util::Config as MemseqConfig; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Config { - /// The appd configuration - #[serde(default)] - pub appd: appd::Config, - - /// The bridge configuration - #[serde(default)] - pub bridge: bridge::Config, - - /// The movement-celestia-da-light-node configuration - #[serde(default)] - pub da_light_node: da_light_node::Config, - - /// Whether to force a new chain - #[serde(default = "default_celestia_force_new_chain")] - pub celestia_force_new_chain: bool, - - /// The memseq configuration - #[serde(default)] - pub memseq: MemseqConfig, - - #[serde(default = "default_da_light_node_is_initial")] - pub da_light_node_is_initial: bool, - - /// The access control config - #[serde(default)] - pub access_control: WhitelistConfig, - - /// The digest store configuration - #[serde(default)] - pub digest_store: digest_store::Config, -} - -impl Default for Config { - fn default() -> Self { - Self { - appd: appd::Config::default(), - bridge: bridge::Config::default(), - da_light_node: da_light_node::Config::default(), - celestia_force_new_chain: default_celestia_force_new_chain(), - memseq: MemseqConfig::default(), - da_light_node_is_initial: default_da_light_node_is_initial(), - access_control: WhitelistConfig::default(), - digest_store: digest_store::Config::default(), - } - } -} diff --git a/protocol-units/da/movement/protocol/util/src/config/mod.rs b/protocol-units/da/movement/protocol/util/src/config/mod.rs index e6c35b1ff..35771e8ae 100644 --- a/protocol-units/da/movement/protocol/util/src/config/mod.rs +++ b/protocol-units/da/movement/protocol/util/src/config/mod.rs @@ -1,7 +1,17 @@ +pub mod appd; +pub mod bridge; +pub mod da_light_node; +pub mod default; +pub mod digest_store; + +use self::default::{default_celestia_force_new_chain, default_da_light_node_is_initial}; + use anyhow::Context; +use aptos_account_whitelist::config::Config as WhitelistConfig; use aptos_types::account_address::AccountAddress; use celestia_rpc::Client; use celestia_types::nmt::Namespace; +use memseq_util::Config as MemseqConfig; use movement_signer::cryptography::{secp256k1::Secp256k1, Curve}; use movement_signer_loader::{Load, LoadedSigner}; use serde::{Deserialize, Serialize}; @@ -9,153 +19,113 @@ use std::collections::HashSet; use std::future::Future; use std::path::PathBuf; -pub mod common; -pub mod local; +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum Network { + Local, + Arabica, + Mocha, + Mainnet, +} #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum Config { - Local(local::Config), - Arabica(local::Config), - Mocha(local::Config), +pub struct Config { + pub network: Network, + + /// The appd configuration + #[serde(default)] + pub appd: appd::Config, + + /// The bridge configuration + #[serde(default)] + pub bridge: bridge::Config, + + /// The movement-celestia-da-light-node configuration + #[serde(default)] + pub da_light_node: da_light_node::Config, + + /// Whether to force a new chain + #[serde(default = "default_celestia_force_new_chain")] + pub celestia_force_new_chain: bool, + + /// The memseq configuration + #[serde(default)] + pub memseq: MemseqConfig, + + #[serde(default = "default_da_light_node_is_initial")] + pub da_light_node_is_initial: bool, + + /// The access control config + #[serde(default)] + pub access_control: WhitelistConfig, + + /// The digest store configuration + #[serde(default)] + pub digest_store: digest_store::Config, } impl Default for Config { fn default() -> Self { - std::env::var("CELESTIA_NETWORK").map_or_else( - |_| Config::Local(local::Config::default()), + let network = std::env::var("CELESTIA_NETWORK").map_or_else( + |_| Network::Local, |network| match network.as_str() { - "arabica" => Config::Arabica(local::Config::default()), - "mocha" => Config::Mocha(local::Config::default()), - _ => Config::Local(local::Config::default()), + "mainnet" => Network::Mainnet, + "arabica" => Network::Arabica, + "mocha" => Network::Mocha, + _ => Network::Local, }, - ) + ); + Self { + network, + appd: appd::Config::default(), + bridge: bridge::Config::default(), + da_light_node: da_light_node::Config::default(), + celestia_force_new_chain: default_celestia_force_new_chain(), + memseq: MemseqConfig::default(), + da_light_node_is_initial: default_da_light_node_is_initial(), + access_control: WhitelistConfig::default(), + digest_store: digest_store::Config::default(), + } } } impl Config { /// Connects to a Celestia node using the config pub async fn connect_celestia(&self) -> Result { - match self { - Config::Local(local) => { - let celestia_node_url = format!( - "{}://{}:{}", - local.appd.celestia_websocket_connection_protocol, - local.appd.celestia_websocket_connection_hostname, - local.appd.celestia_websocket_connection_port - ); - let celestia_auth_token = local.appd.celestia_auth_token.clone().context( - "Failed to get Celestia auth token from config. This is required for connecting to Celestia.", - )?; - - let client = Client::new(&celestia_node_url, Some(&celestia_auth_token)) - .await - .map_err(|e| { - anyhow::anyhow!( - "Failed to connect to Celestia client at {:?}: {}", - celestia_node_url, - e - ) - })?; - - Ok(client) - } - Config::Arabica(local) => { - // arabica is also local for now - let celestia_node_url = format!( - "{}://{}:{}", - local.appd.celestia_websocket_connection_protocol, - local.appd.celestia_websocket_connection_hostname, - local.appd.celestia_websocket_connection_port - ); - let celestia_auth_token = local.appd.celestia_auth_token.clone().context( - "Failed to get Celestia auth token from config. This is required for connecting to Celestia.", - )?; - - let client = Client::new(&celestia_node_url, Some(&celestia_auth_token)) - .await - .map_err(|e| { - anyhow::anyhow!( - "Failed to connect to Celestia client at {:?}: {}", - celestia_node_url, - e - ) - })?; - - Ok(client) - } - Config::Mocha(local) => { - // mocha is also local for now - let celestia_node_url = format!( - "{}://{}:{}", - local.appd.celestia_websocket_connection_protocol, - local.appd.celestia_websocket_connection_hostname, - local.appd.celestia_websocket_connection_port - ); - let celestia_auth_token = local.appd.celestia_auth_token.clone().context( - "Failed to get Celestia auth token from config. This is required for connecting to Celestia.", - )?; - - let client = Client::new(&celestia_node_url, Some(&celestia_auth_token)) - .await - .map_err(|e| { - anyhow::anyhow!( - "Failed to connect to Celestia client at {:?}: {}", - celestia_node_url, - e - ) - })?; - - Ok(client) - } - } + let celestia_node_url = self.appd.celestia_websocket_url(); + let celestia_auth_token = self.appd.celestia_auth_token.clone().context( + "Failed to get Celestia auth token from config. This is required for connecting to Celestia.", + )?; + + let client = + Client::new(&celestia_node_url, Some(&celestia_auth_token)).await.map_err(|e| { + anyhow::anyhow!( + "Failed to connect to Celestia client at {:?}: {}", + celestia_node_url, + e + ) + })?; + + Ok(client) } /// Gets the Celestia namespace pub fn celestia_namespace(&self) -> Namespace { - match self { - Config::Local(local) => local.appd.celestia_namespace.clone(), - Config::Arabica(local) => local.appd.celestia_namespace.clone(), - Config::Mocha(local) => local.appd.celestia_namespace.clone(), - } + self.appd.celestia_namespace.clone() } /// Gets M1 DA Light Node connection protocol pub fn movement_da_light_node_connection_protocol(&self) -> String { - match self { - Config::Local(local) => { - local.da_light_node.movement_da_light_node_connection_protocol.clone() - } - Config::Arabica(local) => { - local.da_light_node.movement_da_light_node_connection_protocol.clone() - } - Config::Mocha(local) => { - local.da_light_node.movement_da_light_node_connection_protocol.clone() - } - } + self.da_light_node.movement_da_light_node_connection_protocol.clone() } /// Gets M1 DA Light Node listen hostname pub fn movement_da_light_node_listen_hostname(&self) -> String { - match self { - Config::Local(local) => { - local.da_light_node.movement_da_light_node_listen_hostname.clone() - } - Config::Arabica(local) => { - local.da_light_node.movement_da_light_node_listen_hostname.clone() - } - Config::Mocha(local) => { - local.da_light_node.movement_da_light_node_listen_hostname.clone() - } - } + self.da_light_node.movement_da_light_node_listen_hostname.clone() } /// Gets M1 DA Light Node listen port pub fn movement_da_light_node_listen_port(&self) -> u16 { - match self { - Config::Local(local) => local.da_light_node.movement_da_light_node_listen_port, - Config::Arabica(local) => local.da_light_node.movement_da_light_node_listen_port, - Config::Mocha(local) => local.da_light_node.movement_da_light_node_listen_port, - } + self.da_light_node.movement_da_light_node_listen_port } /// Gets M1 DA Light Node service @@ -167,89 +137,41 @@ impl Config { /// Gets M1 DA Light Node connection hostname pub fn movement_da_light_node_connection_hostname(&self) -> String { - match self { - Config::Local(local) => { - local.da_light_node.movement_da_light_node_connection_hostname.clone() - } - Config::Arabica(local) => { - local.da_light_node.movement_da_light_node_connection_hostname.clone() - } - Config::Mocha(local) => { - local.da_light_node.movement_da_light_node_connection_hostname.clone() - } - } + self.da_light_node.movement_da_light_node_connection_hostname.clone() } /// Gets M1 DA Light Node connection port pub fn movement_da_light_node_connection_port(&self) -> u16 { - match self { - Config::Local(local) => local.da_light_node.movement_da_light_node_connection_port, - Config::Arabica(local) => local.da_light_node.movement_da_light_node_connection_port, - Config::Mocha(local) => local.da_light_node.movement_da_light_node_connection_port, - } + self.da_light_node.movement_da_light_node_connection_port } /// Whether to use HTTP/1.1 for the movement-da-light-node service pub fn movement_da_light_node_http1(&self) -> bool { - match self { - Config::Local(local) => local.da_light_node.movement_da_light_node_http1, - Config::Arabica(local) => local.da_light_node.movement_da_light_node_http1, - Config::Mocha(local) => local.da_light_node.movement_da_light_node_http1, - } + self.da_light_node.movement_da_light_node_http1 } /// Gets the memseq path pub fn try_memseq_path(&self) -> Result { - match self { - Config::Local(local) => local.memseq.sequencer_database_path.clone().context( + self.memseq.sequencer_database_path.clone().context( "Failed to get memseq path from config. This is required for initializing the memseq database.", - ), - Config::Arabica(local) => local.memseq.sequencer_database_path.clone().context( - "Failed to get memseq path from config. This is required for initializing the memseq database.", - ), - Config::Mocha(local) => local.memseq.sequencer_database_path.clone().context( - "Failed to get memseq path from config. This is required for initializing the memseq database.", - ), - } + ) } /// Gets the da signers sec1 keys pub fn da_signers_sec1_keys(&self) -> HashSet { - match self { - Config::Local(local) => local.da_light_node.da_signers.public_keys_hex.clone(), - Config::Arabica(local) => local.da_light_node.da_signers.public_keys_hex.clone(), - Config::Mocha(local) => local.da_light_node.da_signers.public_keys_hex.clone(), - } + self.da_light_node.da_signers.public_keys_hex.clone() } - pub fn try_block_building_parameters(&self) -> Result<(u32, u64), anyhow::Error> { - match self { - Config::Local(local) => { - Ok((local.memseq.memseq_max_block_size, local.memseq.memseq_build_time)) - } - Config::Arabica(local) => { - Ok((local.memseq.memseq_max_block_size, local.memseq.memseq_build_time)) - } - Config::Mocha(local) => { - Ok((local.memseq.memseq_max_block_size, local.memseq.memseq_build_time)) - } - } + pub fn block_building_parameters(&self) -> (u32, u64) { + (self.memseq.memseq_max_block_size, self.memseq.memseq_build_time) } pub fn whitelisted_accounts(&self) -> Result>, anyhow::Error> { - match self { - Config::Local(local) => local.access_control.whitelisted_accounts(), - Config::Arabica(local) => local.access_control.whitelisted_accounts(), - Config::Mocha(local) => local.access_control.whitelisted_accounts(), - } + self.access_control.whitelisted_accounts() } pub fn digest_store_db_path(&self) -> PathBuf { - match self { - Config::Local(local) => local.digest_store.digest_store_db_path.clone(), - Config::Arabica(local) => local.digest_store.digest_store_db_path.clone(), - Config::Mocha(local) => local.digest_store.digest_store_db_path.clone(), - } + self.digest_store.digest_store_db_path.clone() } } @@ -263,35 +185,13 @@ where impl LoadSigner for Config { async fn da_signer(&self) -> Result, anyhow::Error> { - match self { - Config::Local(local) => { - let identifier: Box + Send> = - Box::new(local.da_light_node.da_signers.signer_identifier.clone()); - let signer = identifier - .load() - .await - .map_err(|e| anyhow::anyhow!("failed to load signer: {}", e))?; - Ok(signer) - } - Config::Arabica(local) => { - let identifier: Box + Send> = - Box::new(local.da_light_node.da_signers.signer_identifier.clone()); - let signer = identifier - .load() - .await - .map_err(|e| anyhow::anyhow!("failed to load signer: {}", e))?; - Ok(signer) - } - Config::Mocha(local) => { - let identifier: Box + Send> = - Box::new(local.da_light_node.da_signers.signer_identifier.clone()); - let signer = identifier - .load() - .await - .map_err(|e| anyhow::anyhow!("failed to load signer: {}", e))?; - Ok(signer) - } - } + let identifier: Box + Send> = + Box::new(self.da_light_node.da_signers.signer_identifier.clone()); + let signer = identifier + .load() + .await + .map_err(|e| anyhow::anyhow!("failed to load signer: {}", e))?; + Ok(signer) } } From 39b8d73c19ee3f3fadfd0b7fd3861ebdcd694d36 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Thu, 6 Feb 2025 18:55:41 +0200 Subject: [PATCH 02/35] fix(CI): movement-da-util in unit-tests The crate name has been changed recently. --- .github/workflows/checks-all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks-all.yml b/.github/workflows/checks-all.yml index 87fc54a6d..202f0cb35 100755 --- a/.github/workflows/checks-all.yml +++ b/.github/workflows/checks-all.yml @@ -65,7 +65,7 @@ jobs: -p move-rocks \ -p movement-types \ -p movement-config \ - -p movement-celestia-da-util \ + -p movement-da-util \ -p movement-signer-test movement-full-node-local: From 180cdbdf6075836837f0f91196ab32055f80794c Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Mon, 10 Feb 2025 17:33:38 +0200 Subject: [PATCH 03/35] fix: celestia mainnet URLs --- .../protocol/celestia-runners/src/celestia_light/mainnet.rs | 4 ++-- protocol-units/da/movement/protocol/setup/src/mainnet.rs | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs index a07addc3e..5bee18d8c 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs @@ -11,14 +11,14 @@ impl Mainnet { _dot_movement: dot_movement::DotMovement, _config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { - // celestia light start --core.ip ??? --p2p.network celestia + // celestia light start --core.ip rpc.celestia.pops.one --p2p.network celestia commander::run_command( "celestia", &[ "light", "start", "--core.ip", - "rpc.celestia.pops.one", // FIXME: is this correct? + "rpc.celestia.pops.one", "--p2p.network", "celestia", "--log.level", diff --git a/protocol-units/da/movement/protocol/setup/src/mainnet.rs b/protocol-units/da/movement/protocol/setup/src/mainnet.rs index 640fdddec..bef6be45b 100644 --- a/protocol-units/da/movement/protocol/setup/src/mainnet.rs +++ b/protocol-units/da/movement/protocol/setup/src/mainnet.rs @@ -63,8 +63,7 @@ impl Mainnet { } pub async fn get_da_block_height(&self) -> Result { - todo!("what's this for mainnet?"); - let response = reqwest::get("https://rpc-???.pops.one/block").await?.text().await?; + let response = reqwest::get("https://rpc.celestia.pops.one/block").await?.text().await?; Ok(response.parse().context("Failed to parse the response to a u64.")?) } From 29c04286b0521c4a4fe07d92d194a8016142f14e Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Mon, 10 Feb 2025 17:43:15 +0200 Subject: [PATCH 04/35] feat!(da): celestia light node keyname configuration --- .../src/celestia_light/arabica.rs | 4 +++- .../src/celestia_light/mainnet.rs | 4 +++- .../celestia-runners/src/celestia_light/mocha.rs | 4 +++- .../movement/protocol/util/src/config/light.rs | 16 ++++++++++++++++ .../da/movement/protocol/util/src/config/mod.rs | 7 +++++++ 5 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 protocol-units/da/movement/protocol/util/src/config/light.rs diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs index 648b36e35..664b150bf 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs @@ -9,7 +9,7 @@ impl Arabica { pub async fn run( &self, _dot_movement: dot_movement::DotMovement, - _config: movement_da_util::config::Config, + config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { // celestia light start --core.ip validator-1.celestia-arabica-11.com --p2p.network arabica commander::run_command( @@ -17,6 +17,8 @@ impl Arabica { &[ "light", "start", + "--keyring.keyname", + &config.light.key_name, "--core.ip", "validator-1.celestia-arabica-11.com", "--p2p.network", diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs index 5bee18d8c..354c3a140 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs @@ -9,7 +9,7 @@ impl Mainnet { pub async fn run( &self, _dot_movement: dot_movement::DotMovement, - _config: movement_da_util::config::Config, + config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { // celestia light start --core.ip rpc.celestia.pops.one --p2p.network celestia commander::run_command( @@ -17,6 +17,8 @@ impl Mainnet { &[ "light", "start", + "--keyring.keyname", + &config.light.key_name, "--core.ip", "rpc.celestia.pops.one", "--p2p.network", diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs index 30bd0ada3..1683eb14d 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs @@ -9,7 +9,7 @@ impl Mocha { pub async fn run( &self, _dot_movement: dot_movement::DotMovement, - _config: movement_da_util::config::Config, + config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { // celestia light start --core.ip validator-1.celestia-mocha-11.com --p2p.network mocha commander::run_command( @@ -17,6 +17,8 @@ impl Mocha { &[ "light", "start", + "--keyring.keyname", + &config.light.key_name, "--core.ip", "rpc-mocha.pops.one", "--p2p.network", diff --git a/protocol-units/da/movement/protocol/util/src/config/light.rs b/protocol-units/da/movement/protocol/util/src/config/light.rs new file mode 100644 index 000000000..8360f68b6 --- /dev/null +++ b/protocol-units/da/movement/protocol/util/src/config/light.rs @@ -0,0 +1,16 @@ +use serde::{Deserialize, Serialize}; + +/// Name of the node's signing key in the keyring that is used by default. +pub const DEFAULT_KEY_NAME: &str = "movement_celestia"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Config { + /// Name of the node's signing key in the keyring + pub key_name: String, +} + +impl Default for Config { + fn default() -> Self { + Self { key_name: DEFAULT_KEY_NAME.into() } + } +} diff --git a/protocol-units/da/movement/protocol/util/src/config/mod.rs b/protocol-units/da/movement/protocol/util/src/config/mod.rs index 35771e8ae..5632ded58 100644 --- a/protocol-units/da/movement/protocol/util/src/config/mod.rs +++ b/protocol-units/da/movement/protocol/util/src/config/mod.rs @@ -3,6 +3,7 @@ pub mod bridge; pub mod da_light_node; pub mod default; pub mod digest_store; +pub mod light; use self::default::{default_celestia_force_new_chain, default_da_light_node_is_initial}; @@ -39,6 +40,11 @@ pub struct Config { #[serde(default)] pub bridge: bridge::Config, + /// The celestia light node configuration. + /// Not to be confused by the movement-celestia-da-light-node-configuration. + #[serde(default)] + pub light: light::Config, + /// The movement-celestia-da-light-node configuration #[serde(default)] pub da_light_node: da_light_node::Config, @@ -78,6 +84,7 @@ impl Default for Config { network, appd: appd::Config::default(), bridge: bridge::Config::default(), + light: light::Config::default(), da_light_node: da_light_node::Config::default(), celestia_force_new_chain: default_celestia_force_new_chain(), memseq: MemseqConfig::default(), From e7d94fda1734a6db63aa1f7b272d2278726f5f85 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Mon, 10 Feb 2025 17:46:39 +0200 Subject: [PATCH 05/35] feat: process-compose overlay for celestia-mainnet --- .../process-compose.celestia-mainnet.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 process-compose/movement-full-node/process-compose.celestia-mainnet.yml diff --git a/process-compose/movement-full-node/process-compose.celestia-mainnet.yml b/process-compose/movement-full-node/process-compose.celestia-mainnet.yml new file mode 100644 index 000000000..2e5fce67e --- /dev/null +++ b/process-compose/movement-full-node/process-compose.celestia-mainnet.yml @@ -0,0 +1,18 @@ +version: "3" + +processes: + + setup: + environment: + - "CELESTIA_NETWORK=mainnet" + + celestia-light-node: + command: | + movement-celestia-light + readiness_probe: + initial_delay_seconds: 60 + exec: + command: curl http://0.0.0.0:26658 + depends_on: + setup: + condition: process_healthy \ No newline at end of file From fc7fb79f36b5f82e6148cdaf108cb5a25c5bc310 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Mon, 10 Feb 2025 18:06:22 +0200 Subject: [PATCH 06/35] feat(da): set Celestia light node key name from env --- .../da/movement/protocol/util/src/config/default.rs | 8 ++++++++ .../da/movement/protocol/util/src/config/light.rs | 7 +++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/protocol-units/da/movement/protocol/util/src/config/default.rs b/protocol-units/da/movement/protocol/util/src/config/default.rs index 17cf5acfb..ac380ea85 100644 --- a/protocol-units/da/movement/protocol/util/src/config/default.rs +++ b/protocol-units/da/movement/protocol/util/src/config/default.rs @@ -157,3 +157,11 @@ env_default!(default_da_light_node_is_initial, "MOVEMENT_DA_LIGHT_NODE_IS_INITIA // Whether to use http1 for Movement Light Node Connections env_default!(default_movement_da_light_node_http1, "MOVEMENT_DA_LIGHT_NODE_HTTP1", bool, true); + +// The Celestia light node's signing keyname +env_default!( + default_celestia_light_node_key_name, + "CELESTIA_LIGHT_KEYNAME", + String, + "movement_celestia_light".into() +); diff --git a/protocol-units/da/movement/protocol/util/src/config/light.rs b/protocol-units/da/movement/protocol/util/src/config/light.rs index 8360f68b6..0d4d48da0 100644 --- a/protocol-units/da/movement/protocol/util/src/config/light.rs +++ b/protocol-units/da/movement/protocol/util/src/config/light.rs @@ -1,7 +1,6 @@ -use serde::{Deserialize, Serialize}; +use super::default::default_celestia_light_node_key_name; -/// Name of the node's signing key in the keyring that is used by default. -pub const DEFAULT_KEY_NAME: &str = "movement_celestia"; +use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct Config { @@ -11,6 +10,6 @@ pub struct Config { impl Default for Config { fn default() -> Self { - Self { key_name: DEFAULT_KEY_NAME.into() } + Self { key_name: default_celestia_light_node_key_name() } } } From e60f2e5ae01bbfce5403e5fc252edb8839ae2ff5 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Wed, 12 Feb 2025 13:51:30 +0200 Subject: [PATCH 07/35] chore(da): clean up old celestia files --- .../src/permissioned_signers/mod.rs | 80 --- .../celestia/light-node/src/v1/passthrough.rs | 521 ------------------ .../da/movement/celestia/util/src/ir_blob.rs | 334 ----------- 3 files changed, 935 deletions(-) delete mode 100644 protocol-units/da/movement/celestia/light-node-verifier/src/permissioned_signers/mod.rs delete mode 100644 protocol-units/da/movement/celestia/light-node/src/v1/passthrough.rs delete mode 100644 protocol-units/da/movement/celestia/util/src/ir_blob.rs diff --git a/protocol-units/da/movement/celestia/light-node-verifier/src/permissioned_signers/mod.rs b/protocol-units/da/movement/celestia/light-node-verifier/src/permissioned_signers/mod.rs deleted file mode 100644 index e9ddd728e..000000000 --- a/protocol-units/da/movement/celestia/light-node-verifier/src/permissioned_signers/mod.rs +++ /dev/null @@ -1,80 +0,0 @@ -use crate::{ - celestia::Verifier as CelestiaVerifier, signed::InKnownSignersVerifier, Error, Verified, - VerifierOperations, -}; -use celestia_rpc::Client; -use celestia_types::nmt::Namespace; -use celestia_types::Blob as CelestiaBlob; -use ecdsa::{ - elliptic_curve::{ - generic_array::ArrayLength, - ops::Invert, - point::PointCompression, - sec1::{FromEncodedPoint, ModulusSize, ToEncodedPoint}, - subtle::CtOption, - AffinePoint, CurveArithmetic, FieldBytesSize, PrimeCurve, Scalar, - }, - hazmat::{DigestPrimitive, SignPrimitive, VerifyPrimitive}, - SignatureSize, -}; -use movement_celestia_da_util::ir_blob::IntermediateBlobRepresentation; -use std::sync::Arc; - -/// A verifier of Celestia blobs for permissioned signers -#[derive(Clone)] -pub struct Verifier -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - /// The Celestia veifier - pub celestia: CelestiaVerifier, - /// The verifier for known signers - pub known_signers: InKnownSignersVerifier, -} - -impl Verifier -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - pub fn new( - celestia_client: Arc, - celestia_namespace: Namespace, - known_signers_sec1_bytes: T, - ) -> Self - where - T: IntoIterator, - T::Item: Into, - { - Self { - celestia: CelestiaVerifier::new(celestia_client, celestia_namespace), - known_signers: InKnownSignersVerifier::new(known_signers_sec1_bytes), - } - } -} - -#[tonic::async_trait] -impl VerifierOperations for Verifier -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - async fn verify( - &self, - blob: CelestiaBlob, - height: u64, - ) -> Result, Error> { - let verified_blob = self.celestia.verify(blob, height).await?; - self.known_signers.verify(verified_blob.into_inner(), height).await - } -} diff --git a/protocol-units/da/movement/celestia/light-node/src/v1/passthrough.rs b/protocol-units/da/movement/celestia/light-node/src/v1/passthrough.rs deleted file mode 100644 index b74178baf..000000000 --- a/protocol-units/da/movement/celestia/light-node/src/v1/passthrough.rs +++ /dev/null @@ -1,521 +0,0 @@ -use movement_celestia_da_util::ir_blob::IntermediateBlobRepresentation; -use std::fmt::{self, Debug, Formatter}; -use std::sync::Arc; -use tokio_stream::{Stream, StreamExt}; -use tracing::{debug, error, info}; - -use celestia_rpc::{BlobClient, Client, HeaderClient}; -use celestia_types::{nmt::Namespace, Blob as CelestiaBlob, TxConfig}; - -// FIXME: glob imports are bad style -use movement_celestia_da_light_node_verifier::{ - permissioned_signers::Verifier, VerifierOperations, -}; -use movement_celestia_da_util::{ - config::Config, - ir_blob::{celestia::CelestiaIntermediateBlobRepresentation, InnerSignedBlobV1Data}, -}; -use movement_da_light_node_proto::light_node_service_server::LightNodeService; -use movement_da_light_node_proto::*; - -use crate::v1::LightNodeV1Operations; -use ecdsa::{ - elliptic_curve::{ - generic_array::ArrayLength, - ops::Invert, - point::PointCompression, - sec1::{FromEncodedPoint, ModulusSize, ToEncodedPoint}, - subtle::CtOption, - AffinePoint, CurveArithmetic, FieldBytesSize, PrimeCurve, Scalar, - }, - hazmat::{DigestPrimitive, SignPrimitive, VerifyPrimitive}, - SignatureSize, SigningKey, -}; - -#[derive(Clone)] -pub struct LightNodeV1 -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - pub config: Config, - pub celestia_namespace: Namespace, - pub default_client: Arc, - pub verifier: Arc< - Box + Send + Sync>, - >, - pub signing_key: SigningKey, -} - -impl Debug for LightNodeV1 -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("LightNodeV1") - .field("celestia_namespace", &self.config.celestia_namespace()) - .finish() - } -} - -impl LightNodeV1Operations for LightNodeV1 -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - /// Tries to create a new LightNodeV1 instance from the toml config file. - async fn try_from_config(config: Config) -> Result { - let client = Arc::new(config.connect_celestia().await?); - - let signing_key_str = config.da_signing_key(); - let hex_bytes = hex::decode(signing_key_str)?; - - let signing_key = SigningKey::from_bytes(hex_bytes.as_slice().try_into()?) - .map_err(|e| anyhow::anyhow!("Failed to create signing key: {}", e))?; - - Ok(Self { - config: config.clone(), - celestia_namespace: config.celestia_namespace(), - default_client: client.clone(), - verifier: Arc::new(Box::new(Verifier::::new( - client, - config.celestia_namespace(), - config.da_signers_sec1_keys(), - ))), - signing_key, - }) - } - - fn try_service_address(&self) -> Result { - Ok(self.config.movement_da_light_node_service()) - } - - /// Runs background tasks for the LightNodeV1 instance. - async fn run_background_tasks(&self) -> Result<(), anyhow::Error> { - Ok(()) - } -} - -impl LightNodeV1 -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - /// Creates a new signed blob instance with the provided data. - pub fn create_new_celestia_blob(&self, data: Vec) -> Result { - // mark the timestamp as now in milliseconds - let timestamp = chrono::Utc::now().timestamp_micros() as u64; - - // sign the blob data and the timestamp - let data = InnerSignedBlobV1Data::try_new(data, timestamp)?; - let signed_data = data.try_to_sign(&self.signing_key)?; - - // create the celestia blob - CelestiaIntermediateBlobRepresentation(signed_data.into(), self.celestia_namespace.clone()) - .try_into() - } - - /// Submits a CelestiaBlob to the Celestia node. - pub async fn submit_celestia_blob(&self, blob: CelestiaBlob) -> Result { - let config = TxConfig::default(); - // config.with_gas(2); - let height = self.default_client.blob_submit(&[blob], config).await.map_err(|e| { - error!(error = %e, "failed to submit the blob"); - anyhow::anyhow!("Failed submitting the blob: {}", e) - })?; - - Ok(height) - } - - /// Submits Celestia blobs to the Celestia node. - pub async fn submit_celestia_blobs( - &self, - blobs: &[CelestiaBlob], - ) -> Result { - let height = - self.default_client.blob_submit(blobs, TxConfig::default()).await.map_err(|e| { - error!(error = %e, "failed to submit the blobs"); - anyhow::anyhow!("Failed submitting the blob: {}", e) - })?; - - Ok(height) - } - - /// Submits a blob to the Celestia node. - pub async fn submit_blob(&self, data: Vec) -> Result { - let celestia_blob = self.create_new_celestia_blob(data)?; - let height = self.submit_celestia_blob(celestia_blob.clone()).await?; - Ok(Self::celestia_blob_to_blob(celestia_blob, height)?) - } - - /// Gets the blobs at a given height. - pub async fn get_ir_blobs_at_height( - &self, - height: u64, - ) -> Result, anyhow::Error> { - let height = if height == 0 { 1 } else { height }; - match self.default_client.blob_get_all(height, &[self.celestia_namespace]).await { - Err(e) => { - error!(error = %e, "failed to get blobs at height {height}"); - anyhow::bail!(e); - } - Ok(blobs) => { - let blobs = blobs.unwrap_or_default(); - - let mut verified_blobs = Vec::new(); - for blob in blobs { - match self.verifier.verify(blob, height).await { - Ok(verified_blob) => { - let blob = verified_blob.into_inner(); - info!("verified blob at height {}: {}", height, hex::encode(blob.id())); - verified_blobs.push(blob); - } - Err(e) => { - error!(error = %e, "failed to verify blob"); - } - } - } - - Ok(verified_blobs) - } - } - } - - #[tracing::instrument(target = "movement_timing", level = "info", skip(self))] - async fn get_blobs_at_height(&self, height: u64) -> Result, anyhow::Error> { - let ir_blobs = self.get_ir_blobs_at_height(height).await?; - let mut blobs = Vec::new(); - for ir_blob in ir_blobs { - let blob = Self::ir_blob_to_blob(ir_blob, height)?; - // todo: update logging here - blobs.push(blob); - } - Ok(blobs) - } - - /// Streams blobs until it can't get another one in the loop - pub async fn stream_blobs_in_range( - &self, - start_height: u64, - end_height: Option, - ) -> Result< - std::pin::Pin> + Send>>, - anyhow::Error, - > { - let mut height = start_height; - let end_height = end_height.unwrap_or_else(|| u64::MAX); - let me = Arc::new(self.clone()); - - let stream = async_stream::try_stream! { - loop { - if height > end_height { - break; - } - - let blobs = me.get_blobs_at_height(height).await?; - for blob in blobs { - yield blob; - } - height += 1; - } - }; - - Ok(Box::pin(stream) - as std::pin::Pin> + Send>>) - } - - /// Streams the latest blobs that can subscribed to. - async fn stream_blobs_from_height_on( - &self, - start_height: Option, - ) -> Result< - std::pin::Pin> + Send>>, - anyhow::Error, - > { - let start_height = start_height.unwrap_or_else(|| u64::MAX); - let me = Arc::new(self.clone()); - let mut subscription = me.default_client.header_subscribe().await?; - - let stream = async_stream::try_stream! { - let mut first_flag = true; - while let Some(header_res) = subscription.next().await { - - let header = header_res?; - let height = header.height().into(); - - info!("Stream got header: {:?}", header.height()); - - // back fetch the blobs - if first_flag && (height > start_height) { - - let mut blob_stream = me.stream_blobs_in_range(start_height, Some(height)).await?; - - while let Some(blob) = blob_stream.next().await { - - debug!("Stream got blob: {:?}", blob); - - yield blob?; - } - - } - first_flag = false; - - let blobs = me.get_blobs_at_height(height).await?; - for blob in blobs { - - debug!("Stream got blob: {:?}", blob); - - yield blob; - } - } - }; - - Ok(Box::pin(stream) - as std::pin::Pin> + Send>>) - } - - pub fn ir_blob_to_blob( - ir_blob: IntermediateBlobRepresentation, - height: u64, - ) -> Result { - Ok(Blob { - data: ir_blob.blob().to_vec(), - signature: ir_blob.signature().to_vec(), - timestamp: ir_blob.timestamp(), - signer: ir_blob.signer().to_vec(), - blob_id: ir_blob.id().to_vec(), - height, - }) - } - - pub fn celestia_blob_to_blob(blob: CelestiaBlob, height: u64) -> Result { - let ir_blob: IntermediateBlobRepresentation = blob.try_into()?; - - Ok(Blob { - data: ir_blob.blob().to_vec(), - signature: ir_blob.signature().to_vec(), - timestamp: ir_blob.timestamp(), - signer: ir_blob.signer().to_vec(), - blob_id: ir_blob.id().to_vec(), - height, - }) - } - - pub fn blob_to_blob_write_response(blob: Blob) -> Result { - Ok(BlobResponse { blob_type: Some(blob_response::BlobType::PassedThroughBlob(blob)) }) - } - - pub fn blob_to_blob_read_response(blob: Blob) -> Result { - #[cfg(feature = "sequencer")] - { - Ok(BlobResponse { blob_type: Some(blob_response::BlobType::SequencedBlobBlock(blob)) }) - } - - #[cfg(not(feature = "sequencer"))] - { - Ok(BlobResponse { blob_type: Some(blob_response::BlobType::PassedThroughBlob(blob)) }) - } - } -} - -#[tonic::async_trait] -impl LightNodeService for LightNodeV1 -where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, -{ - /// Server streaming response type for the StreamReadFromHeight method. - type StreamReadFromHeightStream = std::pin::Pin< - Box< - dyn Stream> + Send + 'static, - >, - >; - - /// Stream blobs from a specified height or from the latest height. - async fn stream_read_from_height( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - info!("Stream read from height request: {:?}", request); - - let me = Arc::new(self.clone()); - let height = request.into_inner().height; - - let output = async_stream::try_stream! { - - let mut blob_stream = me.stream_blobs_from_height_on(Some(height)).await.map_err(|e| tonic::Status::internal(e.to_string()))?; - - while let Some(blob) = blob_stream.next().await { - let blob = blob.map_err(|e| tonic::Status::internal(e.to_string()))?; - let response = StreamReadFromHeightResponse { - blob : Some(Self::blob_to_blob_read_response(blob).map_err(|e| tonic::Status::internal(e.to_string()))?) - }; - yield response; - } - - info!("Stream read from height closed for height: {}", height); - - }; - - Ok(tonic::Response::new(Box::pin(output) as Self::StreamReadFromHeightStream)) - } - - /// Server streaming response type for the StreamReadLatest method. - type StreamReadLatestStream = std::pin::Pin< - Box> + Send + 'static>, - >; - - /// Stream the latest blobs. - async fn stream_read_latest( - &self, - _request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - let me = Arc::new(self.clone()); - - let output = async_stream::try_stream! { - - let mut blob_stream = me.stream_blobs_from_height_on(None).await.map_err(|e| tonic::Status::internal(e.to_string()))?; - while let Some(blob) = blob_stream.next().await { - let blob = blob.map_err(|e| tonic::Status::internal(e.to_string()))?; - let response = StreamReadLatestResponse { - blob : Some(Self::blob_to_blob_read_response(blob).map_err(|e| tonic::Status::internal(e.to_string()))?) - }; - yield response; - } - - }; - - Ok(tonic::Response::new(Box::pin(output) as Self::StreamReadLatestStream)) - } - /// Server streaming response type for the StreamWriteCelestiaBlob method. - type StreamWriteBlobStream = std::pin::Pin< - Box> + Send + 'static>, - >; - /// Stream blobs out, either individually or in batches. - async fn stream_write_blob( - &self, - request: tonic::Request>, - ) -> std::result::Result, tonic::Status> { - let mut stream = request.into_inner(); - let me = Arc::new(self.clone()); - - let output = async_stream::try_stream! { - - while let Some(request) = stream.next().await { - let request = request?; - let blob_data = request.blob.ok_or(tonic::Status::invalid_argument("No blob in request"))?.data; - - let blob = me.submit_blob(blob_data).await.map_err(|e| tonic::Status::internal(e.to_string()))?; - - let write_response = StreamWriteBlobResponse { - blob : Some(Self::blob_to_blob_read_response(blob).map_err(|e| tonic::Status::internal(e.to_string()))?) - }; - - yield write_response; - - } - }; - - Ok(tonic::Response::new(Box::pin(output) as Self::StreamWriteBlobStream)) - } - /// Read blobs at a specified height. - async fn read_at_height( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - let height = request.into_inner().height; - let blobs = self - .get_blobs_at_height(height) - .await - .map_err(|e| tonic::Status::internal(e.to_string()))?; - - if blobs.is_empty() { - return Err(tonic::Status::not_found("No blobs found at the specified height")); - } - - let mut blob_responses = Vec::new(); - for blob in blobs { - blob_responses.push( - Self::blob_to_blob_read_response(blob) - .map_err(|e| tonic::Status::internal(e.to_string()))?, - ); - } - - Ok(tonic::Response::new(ReadAtHeightResponse { - // map blobs to the response type - blobs: blob_responses, - })) - } - /// Batch read and write operations for efficiency. - async fn batch_read( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - let heights = request.into_inner().heights; - let mut responses = Vec::with_capacity(heights.len()); - for height in heights { - let blobs = self - .get_blobs_at_height(height) - .await - .map_err(|e| tonic::Status::internal(e.to_string()))?; - - if blobs.is_empty() { - return Err(tonic::Status::not_found("No blobs found at the specified height")); - } - - let mut blob_responses = Vec::new(); - for blob in blobs { - blob_responses.push( - Self::blob_to_blob_read_response(blob) - .map_err(|e| tonic::Status::internal(e.to_string()))?, - ); - } - - responses.push(ReadAtHeightResponse { blobs: blob_responses }) - } - - Ok(tonic::Response::new(BatchReadResponse { responses })) - } - - /// Batch write blobs. - async fn batch_write( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - let blobs = request.into_inner().blobs; - let mut responses = Vec::with_capacity(blobs.len()); - for data in blobs { - let blob = self - .submit_blob(data.data) - .await - .map_err(|e| tonic::Status::internal(e.to_string()))?; - responses.push(blob); - } - - let mut blob_responses = Vec::new(); - for blob in responses { - blob_responses.push( - Self::blob_to_blob_write_response(blob) - .map_err(|e| tonic::Status::internal(e.to_string()))?, - ); - } - - Ok(tonic::Response::new(BatchWriteResponse { blobs: blob_responses })) - } -} diff --git a/protocol-units/da/movement/celestia/util/src/ir_blob.rs b/protocol-units/da/movement/celestia/util/src/ir_blob.rs deleted file mode 100644 index 49f42550e..000000000 --- a/protocol-units/da/movement/celestia/util/src/ir_blob.rs +++ /dev/null @@ -1,334 +0,0 @@ -use anyhow::bail; -use ecdsa::{ - elliptic_curve::{ - generic_array::typenum::Unsigned, - generic_array::ArrayLength, - ops::Invert, - point::PointCompression, - sec1::{FromEncodedPoint, ModulusSize, ToEncodedPoint}, - subtle::CtOption, - AffinePoint, CurveArithmetic, FieldBytesSize, PrimeCurve, Scalar, - }, - hazmat::{DigestPrimitive, SignPrimitive, VerifyPrimitive}, - signature::{digest::Digest, DigestVerifier}, - SignatureSize, SigningKey, VerifyingKey, -}; -use serde::{Deserialize, Serialize}; - -/// Maximum allowed length of a blob data field. -pub const MAX_BLOB_LEN: usize = 64 * 1_024 * 1_024; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InnerSignedBlobV1Data { - pub blob: Vec, - pub timestamp: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Id(Vec); - -/// The id for an Ir Blob -impl Id { - pub fn as_slice(&self) -> &[u8] { - self.0.as_slice() - } - - pub fn into_vec(self) -> Vec { - self.0 - } -} - -impl From> for Id { - fn from(id: Vec) -> Self { - Id(id) - } -} - -impl InnerSignedBlobV1Data { - pub fn try_new(blob: Vec, timestamp: u64) -> Result { - if blob.len() > MAX_BLOB_LEN { - bail!("blob length {} is above the limit", blob.len()); - } - Ok(Self { blob, timestamp }) - } - - /// Computes the id of InnerSignedBlobV1Data - pub fn compute_id(&self) -> Id - where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, - { - let mut id_hasher = C::Digest::new(); - id_hasher.update(self.blob.as_slice()); - id_hasher.update(&self.timestamp.to_be_bytes()); - Id(id_hasher.finalize().to_vec()) - } - - pub fn try_to_sign( - self, - signing_key: &SigningKey, - ) -> Result - where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, - { - let id = self.compute_id::(); - let mut hasher = C::Digest::new(); - hasher.update(self.blob.as_slice()); - hasher.update(&self.timestamp.to_be_bytes()); - hasher.update(id.as_slice()); - let prehash = hasher.finalize(); - let prehash_bytes = prehash.as_slice(); - - let (signature, _recovery_id) = signing_key.sign_prehash_recoverable(prehash_bytes)?; - - Ok(InnerSignedBlobV1 { - data: self, - signature: signature.to_vec(), - signer: signing_key.verifying_key().to_sec1_bytes().to_vec(), - id, - }) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InnerSignedBlobV1 { - pub data: InnerSignedBlobV1Data, - pub signature: Vec, - pub signer: Vec, - pub id: Id, -} - -impl InnerSignedBlobV1 { - pub fn try_verify(&self) -> Result<(), anyhow::Error> - where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, - { - let mut hasher = C::Digest::new(); - hasher.update(self.data.blob.as_slice()); - hasher.update(&self.data.timestamp.to_be_bytes()); - hasher.update(self.id.as_slice()); - - let verifying_key = VerifyingKey::::from_sec1_bytes(self.signer.as_slice())?; - if self.signature.len() != SignatureSize::::to_usize() { - return Err(anyhow::anyhow!("invalid signature length")); - } - let signature = ecdsa::Signature::from_bytes(self.signature.as_slice().into())?; - - match verifying_key.verify_digest(hasher, &signature) { - Ok(_) => Ok(()), - Err(_) => Err(anyhow::anyhow!("Failed to verify signature")), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum IntermediateBlobRepresentation { - SignedV1(InnerSignedBlobV1), -} - -impl From for IntermediateBlobRepresentation { - fn from(inner: InnerSignedBlobV1) -> Self { - IntermediateBlobRepresentation::SignedV1(inner) - } -} - -impl IntermediateBlobRepresentation { - pub fn blob(&self) -> &[u8] { - match self { - IntermediateBlobRepresentation::SignedV1(inner) => inner.data.blob.as_slice(), - } - } - - pub fn signature(&self) -> &[u8] { - match self { - IntermediateBlobRepresentation::SignedV1(inner) => inner.signature.as_slice(), - } - } - - pub fn timestamp(&self) -> u64 { - match self { - IntermediateBlobRepresentation::SignedV1(inner) => inner.data.timestamp, - } - } - - pub fn signer(&self) -> &[u8] { - match self { - IntermediateBlobRepresentation::SignedV1(inner) => inner.signer.as_slice(), - } - } - - pub fn signer_hex(&self) -> String { - hex::encode(self.signer()) - } - - pub fn id(&self) -> &[u8] { - match self { - IntermediateBlobRepresentation::SignedV1(inner) => inner.id.as_slice(), - } - } - - pub fn verify_signature(&self) -> Result<(), anyhow::Error> - where - C: PrimeCurve + CurveArithmetic + DigestPrimitive + PointCompression, - Scalar: Invert>> + SignPrimitive, - SignatureSize: ArrayLength, - AffinePoint: FromEncodedPoint + ToEncodedPoint + VerifyPrimitive, - FieldBytesSize: ModulusSize, - { - match self { - IntermediateBlobRepresentation::SignedV1(inner) => inner.try_verify::(), - } - } -} - -#[cfg(test)] -pub mod test { - - use super::*; - - #[test] - fn test_cannot_change_id_and_verify() -> Result<(), anyhow::Error> { - let blob = InnerSignedBlobV1Data::try_new(vec![1, 2, 3], 123).unwrap(); - let signing_key = SigningKey::::random(&mut rand::thread_rng()); - let signed_blob = blob.try_to_sign(&signing_key)?; - - let mut changed_blob = signed_blob.clone(); - changed_blob.id = Id(vec![1, 2, 3, 4]); - - assert!(changed_blob.try_verify::().is_err()); - - Ok(()) - } - - #[test] - fn poc_verify_does_not_panic_on_wrong_signature_len() -> Result<(), anyhow::Error> { - let s = InnerSignedBlobV1 { - data: InnerSignedBlobV1Data::try_new(vec![1, 2, 3], 123).unwrap(), - signature: vec![], - signer: vec![ - 2, 130, 130, 130, 130, 130, 130, 130, 82, 130, 130, 130, 130, 255, 255, 130, 130, - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - ], - id: Id(vec![1, 2, 3, 4]), - }; - - s.try_verify::().unwrap_err(); - - Ok(()) - } -} - -pub mod celestia { - use super::{IntermediateBlobRepresentation, MAX_BLOB_LEN}; - use anyhow::Context; - use celestia_types::{consts::appconsts::AppVersion, nmt::Namespace, Blob as CelestiaBlob}; - - impl TryFrom for IntermediateBlobRepresentation { - type Error = anyhow::Error; - - // todo: it would be nice to have this be self describing over the compression and serialization format - fn try_from(blob: CelestiaBlob) -> Result { - // decompress the blob and deserialize the data with bcs - let mut decoder = zstd::Decoder::with_buffer(blob.data.as_slice())?; - let blob = bcs::de::Builder::new() - .max_sequence_length(MAX_BLOB_LEN) - .deserialize_reader(&mut decoder) - .context("failed to deserialize blob")?; - - Ok(blob) - } - } - - pub struct CelestiaIntermediateBlobRepresentation( - pub IntermediateBlobRepresentation, - pub Namespace, - ); - - /// Tries to form a CelestiaBlob from a CelestiaIntermediateBlobRepresentation - impl TryFrom for CelestiaBlob { - type Error = anyhow::Error; - - fn try_from(ir_blob: CelestiaIntermediateBlobRepresentation) -> Result { - // Extract the inner blob and namespace - let CelestiaIntermediateBlobRepresentation(ir_blob, namespace) = ir_blob; - - let mut encoder = - zstd::Encoder::new(vec![], 0).context("failed to initialize zstd encoder")?; - - // Serialize the inner blob with bcs and compress with zstd - bcs::serialize_into(&mut encoder, &ir_blob).context("failed to serialize blob")?; - - let compressed_blob = - encoder.finish().context("failed to finish compression of blob")?; - - // Construct the final CelestiaBlob by assigning the compressed data - // and associating it with the provided namespace - Ok(CelestiaBlob::new(namespace, compressed_blob, AppVersion::V2) - .map_err(|e| anyhow::anyhow!(e))?) - } - } - - #[cfg(test)] - mod tests { - use super::*; - use crate::ir_blob::{InnerSignedBlobV1, InnerSignedBlobV1Data}; - - #[test] - fn zstd_bomb() -> anyhow::Result<()> { - // MAGIC + header with max window size - let mut bomb = vec![0x28, 0xb5, 0x2f, 0xfd, 0x0, 0x7f]; - let n_blocks = 0x530000; - for _ in 0..n_blocks { - // RLE block encoding 0xff byte repeated 0x8000 times - bomb.extend([0x02, 0x00, 0x10, 0xff]); - } - // Block to finish the data - bomb.extend(&[0x01, 0x00, 0x00]); - // Check that we fit in celestia limits - assert!(bomb.len() < 0x1_500_000); - - let blob = - CelestiaBlob::new(Namespace::new_v0(b"movement").unwrap(), bomb, AppVersion::V2)?; - >::try_into(blob).unwrap_err(); - Ok(()) - } - - fn dummy_ir_blob(len: usize) -> CelestiaIntermediateBlobRepresentation { - let blob_data = InnerSignedBlobV1Data { blob: vec![0; len], timestamp: 1733879282 }; - // It's no fun to compute -- not Kraftwerk - let test_blob = InnerSignedBlobV1 { - data: blob_data, - signature: vec![0xfa; 64], - signer: vec![0xaf; 32], - id: vec![0xad; 32].into(), - }; - CelestiaIntermediateBlobRepresentation( - test_blob.into(), - Namespace::new_v0(b"movement").unwrap(), - ) - } - - #[test] - #[ignore = "allocates, compresses, and decompresses 2 GiB of data"] - fn blob_size_limit_imposed_by_bcs() -> anyhow::Result<()> { - CelestiaBlob::try_from(dummy_ir_blob(bcs::MAX_SEQUENCE_LENGTH + 1)) - .expect_err("should be rejected"); - - let celestia_blob: CelestiaBlob = dummy_ir_blob(bcs::MAX_SEQUENCE_LENGTH).try_into()?; - let blob_ir: IntermediateBlobRepresentation = celestia_blob.try_into()?; - assert_eq!(blob_ir.blob().len(), bcs::MAX_SEQUENCE_LENGTH); - Ok(()) - } - } -} From 4806157be8880a47d72c06a23c5a4774938c14b9 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Wed, 12 Feb 2025 17:41:17 +0200 Subject: [PATCH 08/35] fix(da): --keyring.backend to celestia light init When "celestia light init ..." is run in setup, the options need to include --keyring.backend. --- .../da/movement/protocol/setup/src/arabica.rs | 10 ++-- .../da/movement/protocol/setup/src/mainnet.rs | 52 ++----------------- .../da/movement/protocol/setup/src/mocha.rs | 52 ++----------------- 3 files changed, 17 insertions(+), 97 deletions(-) diff --git a/protocol-units/da/movement/protocol/setup/src/arabica.rs b/protocol-units/da/movement/protocol/setup/src/arabica.rs index 5b54f45e4..e0e65fd8f 100644 --- a/protocol-units/da/movement/protocol/setup/src/arabica.rs +++ b/protocol-units/da/movement/protocol/setup/src/arabica.rs @@ -15,14 +15,14 @@ impl Arabica { pub async fn get_arabica_11_address(&self) -> Result { // get the json from celkey - // cel-key list --node.type light --keyring-backend test --p2p.network arabica --output json + // cel-key list --node.type light --keyring.backend test --p2p.network arabica --output json let json_string = run_command( "cel-key", &[ "list", "--node.type", "light", - "--keyring-backend", + "--keyring.backend", "test", "--p2p.network", "arabica", @@ -57,7 +57,11 @@ impl Arabica { pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { // celestia light init --p2p.network arabica - run_command("celestia", &["light", "init", "--p2p.network", "arabica"]).await?; + run_command( + "celestia", + &["light", "init", "--p2p.network", "arabica", "--keyring.backend", "test"], + ) + .await?; Ok(()) } diff --git a/protocol-units/da/movement/protocol/setup/src/mainnet.rs b/protocol-units/da/movement/protocol/setup/src/mainnet.rs index bef6be45b..c84dc8ad8 100644 --- a/protocol-units/da/movement/protocol/setup/src/mainnet.rs +++ b/protocol-units/da/movement/protocol/setup/src/mainnet.rs @@ -13,52 +13,14 @@ impl Mainnet { Self } - pub async fn get_mainnet_11_address(&self) -> Result { - // get the json from celkey - // cel-key list --node.type light --keyring-backend test --p2p.network celestia --output json - let json_string = run_command( - "cel-key", - &[ - "list", - "--node.type", - "light", - "--keyring-backend", - "test", - "--p2p.network", - "celestia", - "--output", - "json", - ], + pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { + // celestia light init --p2p.network celestia --keyring.backend test + run_command( + "celestia", + &["light", "init", "--p2p.network", "celestia", "--keyring.backend", "test"], ) .await?; - let json_string = json_string - .lines() - .last() - .context("Failed to get the last line of the json string.")?; - - info!("Mainnet 11 address json: {}", json_string); - - // use serde to convert to json - let json: serde_json::Value = serde_json::from_str(&json_string) - .context("Failed to convert json string to json value for celestia address.")?; - - // q -r '.[0].address' - let address = json - .get(0) - .context("Failed to get the first element of the json array.")? - .get("address") - .context("Failed to get the address field from the json object.")? - .as_str() - .context("Failed to convert the address field to a string.")?; - - Ok(address.to_string()) - } - - pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { - // celestia light init --p2p.network celestia - run_command("celestia", &["light", "init", "--p2p.network", "celestia"]).await?; - Ok(()) } @@ -91,10 +53,6 @@ impl Mainnet { // celestia light init --p2p.network celestia self.celestia_light_init().await?; - // get the mainnet 11 address - let address = self.get_mainnet_11_address().await?; - config.appd.celestia_validator_address.replace(address.clone()); - // get the auth token let auth_token = self.get_auth_token().await?; config.appd.celestia_auth_token.replace(auth_token.clone()); diff --git a/protocol-units/da/movement/protocol/setup/src/mocha.rs b/protocol-units/da/movement/protocol/setup/src/mocha.rs index 7b67f3847..59a5ac966 100644 --- a/protocol-units/da/movement/protocol/setup/src/mocha.rs +++ b/protocol-units/da/movement/protocol/setup/src/mocha.rs @@ -13,51 +13,13 @@ impl Mocha { Self } - pub async fn get_mocha_11_address(&self) -> Result { - // get the json from celkey - // cel-key list --node.type light --keyring-backend test --p2p.network mocha --output json - let json_string = run_command( - "cel-key", - &[ - "list", - "--node.type", - "light", - "--keyring-backend", - "test", - "--p2p.network", - "mocha", - "--output", - "json", - ], - ) - .await?; - - let json_string = json_string - .lines() - .last() - .context("Failed to get the last line of the json string.")?; - - info!("Mocha 11 address json: {}", json_string); - - // use serde to convert to json - let json: serde_json::Value = serde_json::from_str(&json_string) - .context("Failed to convert json string to json value for celestia address.")?; - - // q -r '.[0].address' - let address = json - .get(0) - .context("Failed to get the first element of the json array.")? - .get("address") - .context("Failed to get the address field from the json object.")? - .as_str() - .context("Failed to convert the address field to a string.")?; - - Ok(address.to_string()) - } - pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { // celestia light init --p2p.network mocha - run_command("celestia", &["light", "init", "--p2p.network", "mocha"]).await?; + run_command( + "celestia", + &["light", "init", "--p2p.network", "mocha", "--keyring.backend", "test"], + ) + .await?; Ok(()) } @@ -91,10 +53,6 @@ impl Mocha { // celestia light init --p2p.network mocha self.celestia_light_init().await?; - // get the mocha 11 address - let address = self.get_mocha_11_address().await?; - config.appd.celestia_validator_address.replace(address.clone()); - // get the auth token let auth_token = self.get_auth_token().await?; config.appd.celestia_auth_token.replace(auth_token.clone()); From e1c73f7a37b86ca4c0fa5179dc49b812854e07c4 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Wed, 12 Feb 2025 17:42:23 +0200 Subject: [PATCH 09/35] feat(da): output namespace in da light node log --- protocol-units/da/movement/protocol/light-node/src/sequencer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol-units/da/movement/protocol/light-node/src/sequencer.rs b/protocol-units/da/movement/protocol/light-node/src/sequencer.rs index cfc8876e5..93730c418 100644 --- a/protocol-units/da/movement/protocol/light-node/src/sequencer.rs +++ b/protocol-units/da/movement/protocol/light-node/src/sequencer.rs @@ -87,7 +87,7 @@ impl LightNodeRuntime info!("Initializing LightNode in sequencer mode from environment."); let pass_through = LightNodePassThrough::try_from_config(config.clone()).await?; - info!("Initialized pass through for LightNode in sequencer mode."); + info!("Initialized pass through for LightNode in sequencer mode: {pass_through:?}"); let memseq_path = pass_through.config.try_memseq_path()?; info!("Memseq path: {:?}", memseq_path); From 5c4692913c414b2a7f301ecc9a02aba257336d00 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Wed, 12 Feb 2025 17:42:58 +0200 Subject: [PATCH 10/35] fix(da): --keyring.backend for celestia light start --- .../protocol/celestia-runners/src/celestia_light/mainnet.rs | 2 ++ .../protocol/celestia-runners/src/celestia_light/mocha.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs index 354c3a140..205b6bcf5 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs @@ -17,6 +17,8 @@ impl Mainnet { &[ "light", "start", + "--keyring.backend", + "test", "--keyring.keyname", &config.light.key_name, "--core.ip", diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs index 1683eb14d..6cf10c5ce 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs @@ -17,6 +17,8 @@ impl Mocha { &[ "light", "start", + "--keyring.backend", + "test", "--keyring.keyname", &config.light.key_name, "--core.ip", From fbe04d12a461e92ce3dbcf285326111a9934c3c8 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Thu, 13 Feb 2025 15:23:59 +0200 Subject: [PATCH 11/35] fix(da): don't ignore celestia RPC errors --- protocol-units/da/movement/protocol/da/src/lib.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/protocol-units/da/movement/protocol/da/src/lib.rs b/protocol-units/da/movement/protocol/da/src/lib.rs index a28cf7779..b1f802d23 100644 --- a/protocol-units/da/movement/protocol/da/src/lib.rs +++ b/protocol-units/da/movement/protocol/da/src/lib.rs @@ -70,16 +70,7 @@ where &self, height: u64, ) -> Pin>, DaError>> + Send + '_>> { - Box::pin(async move { - let result = self.get_da_blobs_at_height(height).await; - match result { - Ok(blobs) => Ok(blobs), - Err(e) => { - warn!("failed to get blobs at height: {}", e); - Ok(vec![]) - } - } - }) + self.get_da_blobs_at_height(height) } fn stream_certificates( From b955c7c6f80988b5b738a344583802540e3ba643 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Fri, 14 Feb 2025 12:02:28 +0200 Subject: [PATCH 12/35] ci: cut out movement-signer-test The crate is not currently listed in the workspace, its inclusion impeded by a dependency problem. --- .github/workflows/checks-all.yml | 4 ++-- Cargo.toml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks-all.yml b/.github/workflows/checks-all.yml index 202f0cb35..f77ffc58b 100755 --- a/.github/workflows/checks-all.yml +++ b/.github/workflows/checks-all.yml @@ -65,8 +65,8 @@ jobs: -p move-rocks \ -p movement-types \ -p movement-config \ - -p movement-da-util \ - -p movement-signer-test + -p movement-da-util + # -p movement-signer-test movement-full-node-local: if: github.event.label.name == 'cicd:movement-full-node' || github.ref == 'refs/heads/main' diff --git a/Cargo.toml b/Cargo.toml index c9cfdf7c3..c98930391 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,8 @@ members = [ "util/signing/integrations/aptos", "util/signing/providers/aws-kms", "util/signing/providers/hashicorp-vault", + # FIXME: add after https://github.com/movementlabsxyz/aptos-core/pull/141 + #"util/signing/testing", "demo/hsm", "protocol-units/execution/maptos/framework/releases/*", "protocol-units/execution/maptos/framework/migrations/*", From 0d8d851415c484faa0d3342fbc864ec0ada429ea Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Mon, 17 Feb 2025 13:43:14 +0200 Subject: [PATCH 13/35] fix(da): streamline random celestia ID generation --- .../da/movement/protocol/setup/src/common/celestia.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs index 600f97fdd..1a27c507f 100644 --- a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs +++ b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs @@ -6,19 +6,17 @@ use movement_da_util::config::Config; use rand::Rng; use tracing::info; -pub fn random_hex(bytes: usize) -> String { +fn random_10_bytes() -> [u8; 10] { let mut rng = rand::thread_rng(); - let random_bytes: Vec = (0..bytes).map(|_| rng.gen()).collect(); - hex::encode(random_bytes) + rng.gen() } pub fn random_chain_id() -> String { - random_hex(10) + hex::encode(random_10_bytes()) } pub fn random_namespace() -> Namespace { - let namespace_bytes = random_hex(10); - Namespace::new_v0(&hex::decode(namespace_bytes).unwrap()).unwrap() + Namespace::new_v0(&random_10_bytes()).unwrap() } pub fn initialize_celestia_config( From 95e1d16b05a8ff0fe4c758036a9440a35df05ea5 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Mon, 17 Feb 2025 17:31:56 +0200 Subject: [PATCH 14/35] feat!: set initial celestia synced_height To avoid a costly sync from height 0, make the initial height a config parameter settable by config. --- .../movement-full-node/src/node/da_db.rs | 25 +++++++++++++++++++ .../movement-full-node/src/node/partial.rs | 7 ++++++ .../da/movement/protocol/setup/src/arabica.rs | 3 +++ .../da/movement/protocol/setup/src/mainnet.rs | 3 +++ .../da/movement/protocol/setup/src/mocha.rs | 3 +++ .../movement/protocol/util/src/config/mod.rs | 6 +++++ 6 files changed, 47 insertions(+) diff --git a/networks/movement/movement-full-node/src/node/da_db.rs b/networks/movement/movement-full-node/src/node/da_db.rs index 9dd13f05d..0cfe80dae 100644 --- a/networks/movement/movement-full-node/src/node/da_db.rs +++ b/networks/movement/movement-full-node/src/node/da_db.rs @@ -77,6 +77,7 @@ impl DaDB { Ok(()) } + /// Get the synced height marker stored in the database. pub async fn get_synced_height(&self) -> Result { // This is heavy for this purpose, but progressively the contents of the DA DB will be used for more things let da_db = self.inner.clone(); @@ -97,4 +98,28 @@ impl DaDB { .await??; Ok(height) } + + /// Set the initial value of the synced height, unless a value is stored. + pub async fn initialize_synced_height(&self, min_height: u64) -> Result<(), anyhow::Error> { + // This is heavy for this purpose, but progressively the contents of the DA DB will be used for more things + let da_db = self.inner.clone(); + tokio::task::spawn_blocking(move || { + let cf = da_db + .cf_handle(SYNCED_HEIGHT) + .ok_or(anyhow::anyhow!("No synced_height column family"))?; + let height = da_db + .get_cf(&cf, "synced_height") + .map_err(|e| anyhow::anyhow!("Failed to get synced height: {:?}", e))?; + if height.is_none() { + let height = serde_json::to_string(&min_height) + .map_err(|e| anyhow::anyhow!("Failed to serialize synced height: {:?}", e))?; + da_db + .put_cf(&cf, "synced_height", height) + .map_err(|e| anyhow::anyhow!("Failed to set synced height: {:?}", e))?; + } + Ok::<(), anyhow::Error>(()) + }) + .await??; + Ok(()) + } } diff --git a/networks/movement/movement-full-node/src/node/partial.rs b/networks/movement/movement-full-node/src/node/partial.rs index ceab1d07b..765563521 100644 --- a/networks/movement/movement-full-node/src/node/partial.rs +++ b/networks/movement/movement-full-node/src/node/partial.rs @@ -164,6 +164,13 @@ impl MovementPartialNode { let da_db = DaDB::open(&config.da_db.da_db_path).context("Failed to create or get DA DB")?; + // FIXME: the config value is probably misplaced + da_db + .initialize_synced_height( + config.celestia_da_light_node.celestia_da_light_node_config.initial_height, + ) + .await?; + Ok(Self { executor, light_node_client, diff --git a/protocol-units/da/movement/protocol/setup/src/arabica.rs b/protocol-units/da/movement/protocol/setup/src/arabica.rs index e0e65fd8f..87a684a13 100644 --- a/protocol-units/da/movement/protocol/setup/src/arabica.rs +++ b/protocol-units/da/movement/protocol/setup/src/arabica.rs @@ -104,6 +104,9 @@ impl Arabica { let auth_token = self.get_auth_token().await?; config.appd.celestia_auth_token.replace(auth_token.clone()); + // get the initial block height + config.initial_height = self.get_da_block_height().await?; + // create and fund the account self.create_and_fund_account(dot_movement.clone(), config.clone()).await?; diff --git a/protocol-units/da/movement/protocol/setup/src/mainnet.rs b/protocol-units/da/movement/protocol/setup/src/mainnet.rs index c84dc8ad8..debcb4fa1 100644 --- a/protocol-units/da/movement/protocol/setup/src/mainnet.rs +++ b/protocol-units/da/movement/protocol/setup/src/mainnet.rs @@ -57,6 +57,9 @@ impl Mainnet { let auth_token = self.get_auth_token().await?; config.appd.celestia_auth_token.replace(auth_token.clone()); + // get the initial block height + config.initial_height = self.get_da_block_height().await?; + Ok(config) } diff --git a/protocol-units/da/movement/protocol/setup/src/mocha.rs b/protocol-units/da/movement/protocol/setup/src/mocha.rs index 59a5ac966..d08ced3b0 100644 --- a/protocol-units/da/movement/protocol/setup/src/mocha.rs +++ b/protocol-units/da/movement/protocol/setup/src/mocha.rs @@ -57,6 +57,9 @@ impl Mocha { let auth_token = self.get_auth_token().await?; config.appd.celestia_auth_token.replace(auth_token.clone()); + // get the initial block height + config.initial_height = self.get_da_block_height().await?; + Ok(config) } diff --git a/protocol-units/da/movement/protocol/util/src/config/mod.rs b/protocol-units/da/movement/protocol/util/src/config/mod.rs index 5632ded58..5d756e795 100644 --- a/protocol-units/da/movement/protocol/util/src/config/mod.rs +++ b/protocol-units/da/movement/protocol/util/src/config/mod.rs @@ -60,6 +60,11 @@ pub struct Config { #[serde(default = "default_da_light_node_is_initial")] pub da_light_node_is_initial: bool, + /// The height to begin fetching blobs from. Filled by querying the network + /// during the initial setup. + #[serde(default)] + pub initial_height: u64, + /// The access control config #[serde(default)] pub access_control: WhitelistConfig, @@ -89,6 +94,7 @@ impl Default for Config { celestia_force_new_chain: default_celestia_force_new_chain(), memseq: MemseqConfig::default(), da_light_node_is_initial: default_da_light_node_is_initial(), + initial_height: 0, access_control: WhitelistConfig::default(), digest_store: digest_store::Config::default(), } From 2ef3ca938bc9da30dfd6799b7460b5efb135091a Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Tue, 18 Feb 2025 18:14:30 +0200 Subject: [PATCH 15/35] fix(da): fetch current block height in setup Make the block height fetching routines do the real thing. --- .../da/movement/protocol/setup/src/arabica.rs | 5 +-- .../protocol/setup/src/common/celestia.rs | 32 +++++++++++++++++++ .../da/movement/protocol/setup/src/mainnet.rs | 5 +-- .../da/movement/protocol/setup/src/mocha.rs | 5 +-- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/protocol-units/da/movement/protocol/setup/src/arabica.rs b/protocol-units/da/movement/protocol/setup/src/arabica.rs index 87a684a13..f2abf85d2 100644 --- a/protocol-units/da/movement/protocol/setup/src/arabica.rs +++ b/protocol-units/da/movement/protocol/setup/src/arabica.rs @@ -67,10 +67,7 @@ impl Arabica { } pub async fn get_da_block_height(&self) -> Result { - let response = - reqwest::get("https://rpc.celestia-arabica-11.com/block").await?.text().await?; - - Ok(response.parse().context("Failed to parse the response to a u64.")?) + common::celestia::current_block_height("https://rpc.celestia-arabica-11.com").await } pub async fn get_auth_token(&self) -> Result { diff --git a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs index 1a27c507f..b0555def5 100644 --- a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs +++ b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs @@ -89,3 +89,35 @@ pub async fn make_dirs( Ok(config) } + +/// Retrieves the current block height from Celestia RPC +pub async fn current_block_height(rpc: &str) -> Result { + // Request the Tendermint JSON-RPC header endpoint + let response = reqwest::get(String::from(rpc) + "/header").await?.text().await?; + + // use serde to convert to json + let json: serde_json::Value = + serde_json::from_str(&response).context("Failed to parse header response as JSON")?; + + let jsonrpc_version = json + .get("jsonrpc") + .context("response is not JSON-RPC")? + .as_str() + .context("invalid jsonrpc field value")?; + if jsonrpc_version != "2.0" { + anyhow::bail!("unexpected JSON-RPC version {jsonrpc_version}"); + } + + // .result.header.height + let height = json + .get("result") + .context("no result field")? + .get("header") + .context("missing header field")? + .get("height") + .context("missing height field")? + .as_str() + .context("expected string value of the height field")? + .parse()?; + Ok(height) +} diff --git a/protocol-units/da/movement/protocol/setup/src/mainnet.rs b/protocol-units/da/movement/protocol/setup/src/mainnet.rs index debcb4fa1..50c41c460 100644 --- a/protocol-units/da/movement/protocol/setup/src/mainnet.rs +++ b/protocol-units/da/movement/protocol/setup/src/mainnet.rs @@ -1,5 +1,4 @@ use crate::common; -use anyhow::Context; use commander::run_command; use dot_movement::DotMovement; use movement_da_util::config::Config; @@ -25,9 +24,7 @@ impl Mainnet { } pub async fn get_da_block_height(&self) -> Result { - let response = reqwest::get("https://rpc.celestia.pops.one/block").await?.text().await?; - - Ok(response.parse().context("Failed to parse the response to a u64.")?) + common::celestia::current_block_height("https://rpc.celestia.pops.one").await } pub async fn get_auth_token(&self) -> Result { diff --git a/protocol-units/da/movement/protocol/setup/src/mocha.rs b/protocol-units/da/movement/protocol/setup/src/mocha.rs index d08ced3b0..d0672c382 100644 --- a/protocol-units/da/movement/protocol/setup/src/mocha.rs +++ b/protocol-units/da/movement/protocol/setup/src/mocha.rs @@ -1,5 +1,4 @@ use crate::common; -use anyhow::Context; use commander::run_command; use dot_movement::DotMovement; use movement_da_util::config::Config; @@ -25,9 +24,7 @@ impl Mocha { } pub async fn get_da_block_height(&self) -> Result { - let response = reqwest::get("https://rpc-mocha.pops.one/block").await?.text().await?; - - Ok(response.parse().context("Failed to parse the response to a u64.")?) + common::celestia::current_block_height("https://rpc-mocha.pops.one").await } pub async fn get_auth_token(&self) -> Result { From 90a44186aaf4852e91b5e08c4ac5900573df11d8 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Tue, 18 Feb 2025 18:16:50 +0200 Subject: [PATCH 16/35] fix(da): force_new_chain to false when well-known If the Celestia network is not local, don't generate a random chain ID. --- protocol-units/da/movement/protocol/util/src/config/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/protocol-units/da/movement/protocol/util/src/config/mod.rs b/protocol-units/da/movement/protocol/util/src/config/mod.rs index 5d756e795..81936fd10 100644 --- a/protocol-units/da/movement/protocol/util/src/config/mod.rs +++ b/protocol-units/da/movement/protocol/util/src/config/mod.rs @@ -85,13 +85,15 @@ impl Default for Config { _ => Network::Local, }, ); + let celestia_force_new_chain = + if network == Network::Local { default_celestia_force_new_chain() } else { false }; Self { network, appd: appd::Config::default(), bridge: bridge::Config::default(), light: light::Config::default(), da_light_node: da_light_node::Config::default(), - celestia_force_new_chain: default_celestia_force_new_chain(), + celestia_force_new_chain, memseq: MemseqConfig::default(), da_light_node_is_initial: default_da_light_node_is_initial(), initial_height: 0, From 186730f1dfa6f6ebaa40b233d403fca896296dc2 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Wed, 19 Feb 2025 14:16:11 +0200 Subject: [PATCH 17/35] fix(da): saner parsing of CELESTIA_NAMESPACE env Harmonize the format of the namespace string with the one expected by the celestia CLI: 0x prefix for hex, otherwise base64. No more expecting quoted base64-encoded string as if it were JSON. --- Cargo.lock | 42 ++++--------------- Cargo.toml | 3 +- .../da/movement/protocol/util/Cargo.toml | 2 +- .../protocol/util/src/config/default.rs | 23 ++++++---- 4 files changed, 26 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 309aeeabe..846dd6c7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5024,9 +5024,6 @@ name = "beef" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" -dependencies = [ - "serde", -] [[package]] name = "bellpepper" @@ -5863,7 +5860,7 @@ dependencies = [ "celestia-types", "futures", "http 1.1.0", - "jsonrpsee 0.24.7", + "jsonrpsee", "serde", "thiserror 1.0.69", "tracing", @@ -8914,7 +8911,7 @@ dependencies = [ "aws-config", "aws-sdk-kms", "axum 0.6.20", - "base64 0.13.1", + "base64 0.22.1", "clap 4.5.21", "dotenv", "ed25519 2.2.3", @@ -9749,15 +9746,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jsonrpsee" -version = "0.20.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138572befc78a9793240645926f30161f8b4143d2be18d09e44ed9814bd7ee2c" -dependencies = [ - "jsonrpsee-types 0.20.4", -] - [[package]] name = "jsonrpsee" version = "0.24.7" @@ -9767,7 +9755,7 @@ dependencies = [ "jsonrpsee-core", "jsonrpsee-http-client", "jsonrpsee-proc-macros", - "jsonrpsee-types 0.24.7", + "jsonrpsee-types", "jsonrpsee-ws-client", "tracing", ] @@ -9808,7 +9796,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "http-body-util", - "jsonrpsee-types 0.24.7", + "jsonrpsee-types", "pin-project 1.1.7", "rustc-hash 2.0.0", "serde", @@ -9832,7 +9820,7 @@ dependencies = [ "hyper-rustls 0.27.3", "hyper-util", "jsonrpsee-core", - "jsonrpsee-types 0.24.7", + "jsonrpsee-types", "rustls 0.23.17", "rustls-platform-verifier", "serde", @@ -9857,20 +9845,6 @@ dependencies = [ "syn 2.0.87", ] -[[package]] -name = "jsonrpsee-types" -version = "0.20.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3264e339143fe37ed081953842ee67bfafa99e3b91559bdded6e4abd8fc8535e" -dependencies = [ - "anyhow", - "beef", - "serde", - "serde_json", - "thiserror 1.0.69", - "tracing", -] - [[package]] name = "jsonrpsee-types" version = "0.24.7" @@ -9892,7 +9866,7 @@ dependencies = [ "http 1.1.0", "jsonrpsee-client-transport", "jsonrpsee-core", - "jsonrpsee-types 0.24.7", + "jsonrpsee-types", "url", ] @@ -12033,6 +12007,7 @@ dependencies = [ "aptos-account-whitelist", "aptos-types", "async-stream", + "base64 0.22.1", "bcs 0.1.6 (git+https://github.com/movementlabsxyz/bcs.git?rev=bc16d2d39cabafaabd76173dd1b04b2aa170cf0c)", "celestia-rpc", "celestia-types", @@ -12040,7 +12015,6 @@ dependencies = [ "ecdsa 0.16.9", "godfig", "hex", - "jsonrpsee 0.20.4", "k256", "memseq-util", "movement-da-light-node-proto", @@ -12225,7 +12199,7 @@ version = "0.0.2" dependencies = [ "anyhow", "async-trait", - "base64 0.13.1", + "base64 0.22.1", "dotenv", "movement-signer", "tokio", diff --git a/Cargo.toml b/Cargo.toml index c98930391..60b843a12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -153,7 +153,6 @@ movement-signing-eth = { path = "util/signing/integrations/eth" } vaultrs = { version = "0.7.3" } aws-sdk-kms = "1.51.0" google-cloud-kms = "0.6.0" -base64 = { version = "0.13.0" } # Serialization and Deserialization serde = "1.0" @@ -268,6 +267,7 @@ async-stream = "0.3.0" async-trait = "0.1.71" async-recursion = "1.1.1" auto_impl = "1.2.0" +base64 = { version = "0.22.1" } blake3 = { version = "1.4.0", features = ["traits-preview"] } bytes = { version = "1.2.1", default-features = false } chrono = "0.4.37" @@ -291,7 +291,6 @@ k256 = { version = "0.13.3" } keccak-hash = "0.10.0" itertools = { version = "0.12.1", default-features = false } jmt = "0.9.0" -jsonrpsee = { version = "0.20.1", features = ["jsonrpsee-types"] } log = "0.4.21" mirai-annotations = "1.10.1" move-vm-integration-test-helpers = { path = "test-helpers/move-vm-integration-test-helpers" } diff --git a/protocol-units/da/movement/protocol/util/Cargo.toml b/protocol-units/da/movement/protocol/util/Cargo.toml index 3b2679889..b42d9f64b 100644 --- a/protocol-units/da/movement/protocol/util/Cargo.toml +++ b/protocol-units/da/movement/protocol/util/Cargo.toml @@ -24,12 +24,12 @@ movement-da-light-node-proto = { workspace = true, features = [] } celestia-rpc = { workspace = true } celestia-types = { workspace = true } anyhow = { workspace = true } +base64 = { workspace = true } hex = { workspace = true } async-stream = { workspace = true } serde_json = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } -jsonrpsee = { workspace = true } dot-movement = { workspace = true } toml = { workspace = true } memseq-util = { workspace = true } diff --git a/protocol-units/da/movement/protocol/util/src/config/default.rs b/protocol-units/da/movement/protocol/util/src/config/default.rs index ac380ea85..f11f4d713 100644 --- a/protocol-units/da/movement/protocol/util/src/config/default.rs +++ b/protocol-units/da/movement/protocol/util/src/config/default.rs @@ -1,6 +1,9 @@ +use base64::prelude::*; use celestia_types::nmt::Namespace; use godfig::env_default; +use std::env; + // The default hostname for the Celestia RPC env_default!( default_celestia_rpc_listen_hostname, @@ -108,14 +111,20 @@ env_default!( // The default Celestia Namespace pub fn default_celestia_namespace() -> Namespace { - match std::env::var("CELESTIA_NAMESPACE") { - Ok(val) => match serde_json::from_str(&val) { - Ok(namespace) => namespace, - // todo: get rid of this unwrap somehow, even though it should never fail - Err(_) => Namespace::new_v0(b"movement").unwrap(), - }, + match env::var("CELESTIA_NAMESPACE") { + Ok(val) => { + if val.starts_with("0x") { + let id = hex::decode(&val[2..]).expect("failed to decode hexadecimal namespace ID"); + Namespace::new_v0(&id).expect("invalid namespace ID") + } else { + let id = + BASE64_STANDARD.decode(&val).expect("failed to decode base64 namespace ID"); + Namespace::from_raw(&id).expect("invalid namespace ID") + } + } // todo: get rid of this unwrap somehow, even though it should never fail - Err(_) => Namespace::new_v0(b"movement").unwrap(), + Err(env::VarError::NotPresent) => Namespace::new_v0(b"movement").unwrap(), + Err(e) => panic!("invalid environment value of CELESTIA_NAMESPACE: {e}"), } } From c23a46d890459acd62881ea99f9841a5f8d3fb76 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Wed, 19 Feb 2025 15:16:24 +0200 Subject: [PATCH 18/35] fix: redundant clone --- networks/movement/setup/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/networks/movement/setup/src/main.rs b/networks/movement/setup/src/main.rs index dbf7ddfdf..3aaf01a6c 100644 --- a/networks/movement/setup/src/main.rs +++ b/networks/movement/setup/src/main.rs @@ -58,7 +58,7 @@ async fn main() -> Result<(), anyhow::Error> { // set up anvil let (config, anvil_join_handle) = Local::default().setup(dot_movement, config).await?; - Ok((Some(config.clone()), anvil_join_handle)) + Ok((Some(config), anvil_join_handle)) }) .await?; From d5b88aaa8de064f0b79c4cb6f20a1a2a3e12ad09 Mon Sep 17 00:00:00 2001 From: Liam Monninger Date: Thu, 20 Feb 2025 18:49:23 -0800 Subject: [PATCH 19/35] feat: celestia-appd loop. --- .../protocol/celestia-runners/src/celestia_appd/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/mod.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/mod.rs index 63a43caee..3c26cb032 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/mod.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_appd/mod.rs @@ -20,7 +20,13 @@ impl Runner for CelestiaAppd { } Network::Arabica => Err(anyhow::anyhow!("Arabica not implemented")), Network::Mocha => Err(anyhow::anyhow!("Mocha not implemented")), - Network::Mainnet => Err(anyhow::anyhow!("Mainnet not implemented")), + Network::Mainnet => { + // loop and sleep over a message that says we are using a direct connection to a trusted Celestia endpoint + loop { + println!("Using a direct connection to a trusted Celestia endpoint"); + std::thread::sleep(std::time::Duration::from_secs(60)); + } + } } } } From 51308b54e94f7c82dff292c733fde6f4635fdb1f Mon Sep 17 00:00:00 2001 From: Liam Monninger Date: Thu, 20 Feb 2025 19:40:10 -0800 Subject: [PATCH 20/35] fix: increase to 16cpvu. --- .github/workflows/build-push-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push-container.yml b/.github/workflows/build-push-container.yml index b7b7cd04a..d32b06e65 100644 --- a/.github/workflows/build-push-container.yml +++ b/.github/workflows/build-push-container.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: architecture: [x86_64, arm64] - runs-on: ${{ matrix.architecture == 'x86_64' && 'buildjet-8vcpu-ubuntu-2204' || 'buildjet-8vcpu-ubuntu-2204-arm' }} + runs-on: ${{ matrix.architecture == 'x86_64' && 'buildjet-16vcpu-ubuntu-2204' || 'buildjet-16vcpu-ubuntu-2204-arm' }} steps: - name: Checkout repository uses: actions/checkout@v4 From c89464488eff6ffe68eef31e47ae285838744440 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Fri, 21 Feb 2025 08:40:45 +0200 Subject: [PATCH 21/35] fix(process-compose): pre-build tests --- .../movement-full-node/process-compose.test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/process-compose/movement-full-node/process-compose.test.yml b/process-compose/movement-full-node/process-compose.test.yml index bfcbca783..cb7fc02fc 100644 --- a/process-compose/movement-full-node/process-compose.test.yml +++ b/process-compose/movement-full-node/process-compose.test.yml @@ -4,10 +4,19 @@ environment: processes: + build-movement-client-tests: + command: | + cargo test --no-run -p movement-client test_example_ + depends_on: + build: + condition: process_completed_successfully + movement-client-tests: command: | cargo test -p movement-client test_example_ depends_on: + build-movement-client-tests: + condition: process_completed_successfully movement-full-node: condition: process_healthy movement-faucet: From 19f0c4565570346a175fd8022750743f5baaca81 Mon Sep 17 00:00:00 2001 From: Liam Monninger Date: Thu, 20 Feb 2025 22:58:34 -0800 Subject: [PATCH 22/35] feat: move into docker compose. --- ...ocker-compose.celestia-mainnet-trusted.yml | 50 +++++++++++++++++++ .../docker-compose.mainnet-leader.yml | 11 ++++ 2 files changed, 61 insertions(+) create mode 100644 docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml create mode 100644 docker/compose/movement-full-node/docker-compose.mainnet-leader.yml diff --git a/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml b/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml new file mode 100644 index 000000000..83daa514c --- /dev/null +++ b/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml @@ -0,0 +1,50 @@ +services: + setup: + environment: + CELESTIA_NAMESPACE: 0x3073030730 + CELESTIA_NETWORK: mainnet + CELESTIA_RPC_CONNECTION_PROTOCOL: https + CELESTIA_RPC_CONNECTION_HOSTNAME: rpc.celestia.pops.one + CELESTIA_RPC_CONNECTION_PORT: 443 + + # turn off underlying da light nodes + celestia-light-node: + image: busybox + container_name: celestia-light-node + command: sleep infinity + environment: + - DOT_MOVEMENT_PATH=/.movement + - CELESTIA_RPC_ADDRESS=celestia-light-node:26657 + volumes: + - ${DOT_MOVEMENT_PATH}:/.movement + depends_on: + setup: + condition: service_healthy + healthcheck: + test: [ "CMD-SHELL", "echo 'health check'" ] + retries: 3 + start_period: 3s + restart: on-failure:3 + + # turn off celestia-light-node-synced + celestia-light-node-synced: + image: busybox + container_name: celestia-light-node-synced + command: echo "No sync check when following." + environment: + - DOT_MOVEMENT_PATH=/.movement + volumes: + - ${DOT_MOVEMENT_PATH}:/.movement + depends_on: + celestia-light-node: + condition: service_healthy + + # turn off movement-celestia-da-light-node + movement-celestia-da-light-node: + image: busybox + container_name: movement-celestia-da-light-node + command: sleep infinity + healthcheck: + test: [ "CMD-SHELL", "echo 'health check'" ] + retries: 3 + start_period: 3s \ No newline at end of file diff --git a/docker/compose/movement-full-node/docker-compose.mainnet-leader.yml b/docker/compose/movement-full-node/docker-compose.mainnet-leader.yml new file mode 100644 index 000000000..dd273568c --- /dev/null +++ b/docker/compose/movement-full-node/docker-compose.mainnet-leader.yml @@ -0,0 +1,11 @@ +services: + + # turn off movement-faucet-service + movement-faucet-service: + image: busybox + container_name: movement-faucet-service + command: sleep infinity + healthcheck: + test: [ "CMD-SHELL", "echo 'health check'" ] + retries: 3 + start_period: 3s \ No newline at end of file From 94f7f0da27b0e072e7913291b3211046ce8a78b4 Mon Sep 17 00:00:00 2001 From: Liam Monninger Date: Thu, 20 Feb 2025 23:11:27 -0800 Subject: [PATCH 23/35] fix: strip out eth. --- .../docker-compose.celestia-mainnet-trusted.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml b/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml index 83daa514c..bd595f58a 100644 --- a/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml +++ b/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml @@ -1,11 +1,24 @@ services: setup: + image: ghcr.io/movementlabsxyz/movement-full-node-setup:${CONTAINER_REV} + container_name: setup environment: CELESTIA_NAMESPACE: 0x3073030730 CELESTIA_NETWORK: mainnet CELESTIA_RPC_CONNECTION_PROTOCOL: https CELESTIA_RPC_CONNECTION_HOSTNAME: rpc.celestia.pops.one CELESTIA_RPC_CONNECTION_PORT: 443 + INDEXER_PROCESSOR_POSTGRES_CONNECTION_STRING: postgres://postgres:password@postgres:5432/postgres + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_REGION: ${AWS_DEFAULT_REGION} + volumes: + - ${DOT_MOVEMENT_PATH}:/.movement + healthcheck: + test: [ "CMD-SHELL", "echo 'health check'" ] + retries: 30 + interval: 10s + timeout: 5s # turn off underlying da light nodes celestia-light-node: From 427b6518a85570c065bb271ee6a5c51784deed49 Mon Sep 17 00:00:00 2001 From: Liam Monninger Date: Fri, 21 Feb 2025 04:07:29 -0800 Subject: [PATCH 24/35] fix: movement celestia still needed. --- .../docker-compose.celestia-mainnet-trusted.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml b/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml index bd595f58a..1b1394875 100644 --- a/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml +++ b/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml @@ -50,14 +50,4 @@ services: - ${DOT_MOVEMENT_PATH}:/.movement depends_on: celestia-light-node: - condition: service_healthy - - # turn off movement-celestia-da-light-node - movement-celestia-da-light-node: - image: busybox - container_name: movement-celestia-da-light-node - command: sleep infinity - healthcheck: - test: [ "CMD-SHELL", "echo 'health check'" ] - retries: 3 - start_period: 3s \ No newline at end of file + condition: service_healthy \ No newline at end of file From d837e6851c91cc3f26c6ca1157b8658406ef93fa Mon Sep 17 00:00:00 2001 From: Liam Monninger Date: Fri, 21 Feb 2025 05:13:32 -0800 Subject: [PATCH 25/35] fix: udpate websocket connection. --- .../docker-compose.celestia-mainnet-trusted.yml | 5 +++++ .../da/movement/protocol/util/src/config/appd.rs | 14 +++++++++++--- .../protocol/util/src/config/da_light_node.rs | 8 +++++++- .../movement/protocol/util/src/config/default.rs | 8 ++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml b/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml index 1b1394875..9cbb40cb8 100644 --- a/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml +++ b/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml @@ -8,6 +8,11 @@ services: CELESTIA_RPC_CONNECTION_PROTOCOL: https CELESTIA_RPC_CONNECTION_HOSTNAME: rpc.celestia.pops.one CELESTIA_RPC_CONNECTION_PORT: 443 + # wss://celestia.cumulo.org.es:443/websocket + CELESTIA_WS_CONNECTION_PROTOCOL: wss + CELESTIA_WS_CONNECTION_HOSTNAME: celestia.cumulo.org.es + CELESTIA_WS_CONNECTION_PORT: 443 + CELESTIA_WS_CONNECTION_PATH: /websocket INDEXER_PROCESSOR_POSTGRES_CONNECTION_STRING: postgres://postgres:password@postgres:5432/postgres AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} diff --git a/protocol-units/da/movement/protocol/util/src/config/appd.rs b/protocol-units/da/movement/protocol/util/src/config/appd.rs index 1be2ab9ae..766872d32 100644 --- a/protocol-units/da/movement/protocol/util/src/config/appd.rs +++ b/protocol-units/da/movement/protocol/util/src/config/appd.rs @@ -2,7 +2,8 @@ use crate::config::default::{ default_celestia_appd_replace_args, default_celestia_appd_use_replace_args, default_celestia_chain_id, default_celestia_namespace, default_celestia_rpc_listen_hostname, default_celestia_rpc_listen_port, default_celestia_websocket_connection_hostname, - default_celestia_websocket_connection_port, default_celestia_websocket_connection_protocol, + default_celestia_websocket_connection_path, default_celestia_websocket_connection_port, + default_celestia_websocket_connection_protocol, }; use celestia_types::nmt::Namespace; @@ -31,6 +32,10 @@ pub struct Config { #[serde(default = "default_celestia_websocket_connection_port")] pub celestia_websocket_connection_port: u16, + /// The path of the Celestia Node websocket + #[serde(default = "default_celestia_websocket_connection_path")] + pub celestia_websocket_connection_path: String, + /// The auth token for the Celestia node pub celestia_auth_token: Option, @@ -69,6 +74,7 @@ impl Default for Config { celestia_websocket_connection_hostname: default_celestia_websocket_connection_hostname( ), celestia_websocket_connection_port: default_celestia_websocket_connection_port(), + celestia_websocket_connection_path: default_celestia_websocket_connection_path(), celestia_chain_id: default_celestia_chain_id(), celestia_auth_token: None, celestia_namespace: default_celestia_namespace(), @@ -84,12 +90,14 @@ impl Config { // FIXME: use a single URL field as the source. The format was introduced by // Sir Tim Berners-Lee in 1994 so you don't have to compose this from // values that need to be set in three different environment variables. + // NOTE: originally, this was not done because there was a need to reuse some parts of the URL but substitute others when running in environments like Docker. pub fn celestia_websocket_url(&self) -> String { format!( - "{}://{}:{}", + "{}://{}:{}{}", self.celestia_websocket_connection_protocol, self.celestia_websocket_connection_hostname, - self.celestia_websocket_connection_port + self.celestia_websocket_connection_port, + self.celestia_websocket_connection_path ) } } diff --git a/protocol-units/da/movement/protocol/util/src/config/da_light_node.rs b/protocol-units/da/movement/protocol/util/src/config/da_light_node.rs index 6009b1994..f95b6b68e 100644 --- a/protocol-units/da/movement/protocol/util/src/config/da_light_node.rs +++ b/protocol-units/da/movement/protocol/util/src/config/da_light_node.rs @@ -1,7 +1,8 @@ use crate::config::default::{ default_celestia_rpc_connection_hostname, default_celestia_rpc_connection_port, default_celestia_rpc_connection_protocol, default_celestia_websocket_connection_hostname, - default_celestia_websocket_connection_port, default_movement_da_light_node_connection_hostname, + default_celestia_websocket_connection_path, default_celestia_websocket_connection_port, + default_movement_da_light_node_connection_hostname, default_movement_da_light_node_connection_port, default_movement_da_light_node_http1, default_movement_da_light_node_listen_hostname, default_movement_da_light_node_listen_port, }; @@ -130,6 +131,10 @@ pub struct Config { #[serde(default = "default_celestia_websocket_connection_port")] pub celestia_websocket_connection_port: u16, + /// The path of the Celestia Node websocket + #[serde(default = "default_celestia_websocket_connection_path")] + pub celestia_websocket_connection_path: String, + // FIXME: disentangle listen config for the light node service // from the connection config to connect to the same service? /// The hostname to listen on for the movement-celestia-da-light-node service @@ -171,6 +176,7 @@ impl Default for Config { celestia_websocket_connection_hostname: default_celestia_websocket_connection_hostname( ), celestia_websocket_connection_port: default_celestia_websocket_connection_port(), + celestia_websocket_connection_path: default_celestia_websocket_connection_path(), movement_da_light_node_listen_hostname: default_movement_da_light_node_listen_hostname( ), movement_da_light_node_listen_port: default_movement_da_light_node_listen_port(), diff --git a/protocol-units/da/movement/protocol/util/src/config/default.rs b/protocol-units/da/movement/protocol/util/src/config/default.rs index f11f4d713..123d2d0ee 100644 --- a/protocol-units/da/movement/protocol/util/src/config/default.rs +++ b/protocol-units/da/movement/protocol/util/src/config/default.rs @@ -69,6 +69,14 @@ env_default!( 26658 ); +// The default Celestia Node websocket path +env_default!( + default_celestia_websocket_connection_path, + "CELESTIA_WEBSOCKET_CONNECTION_PATH", + String, + "".to_string() +); + // The default M1 DA Light Node listen hostname env_default!( default_movement_da_light_node_listen_hostname, From e0392e382c8ee17e97a438c83dfd1bbd770904bb Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Fri, 21 Feb 2025 17:19:50 +0200 Subject: [PATCH 26/35] fix(docker): run light node for celestia-mainnet Also rename the .celestia-mainnet-trusted docker-compose overlay to .celestia-mainnet because we're running the local node. --- ...ocker-compose.celestia-mainnet-trusted.yml | 58 ------------------- .../docker-compose.celestia-mainnet.yml | 47 +++++++++++++++ 2 files changed, 47 insertions(+), 58 deletions(-) delete mode 100644 docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml create mode 100644 docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml diff --git a/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml b/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml deleted file mode 100644 index 9cbb40cb8..000000000 --- a/docker/compose/movement-full-node/docker-compose.celestia-mainnet-trusted.yml +++ /dev/null @@ -1,58 +0,0 @@ -services: - setup: - image: ghcr.io/movementlabsxyz/movement-full-node-setup:${CONTAINER_REV} - container_name: setup - environment: - CELESTIA_NAMESPACE: 0x3073030730 - CELESTIA_NETWORK: mainnet - CELESTIA_RPC_CONNECTION_PROTOCOL: https - CELESTIA_RPC_CONNECTION_HOSTNAME: rpc.celestia.pops.one - CELESTIA_RPC_CONNECTION_PORT: 443 - # wss://celestia.cumulo.org.es:443/websocket - CELESTIA_WS_CONNECTION_PROTOCOL: wss - CELESTIA_WS_CONNECTION_HOSTNAME: celestia.cumulo.org.es - CELESTIA_WS_CONNECTION_PORT: 443 - CELESTIA_WS_CONNECTION_PATH: /websocket - INDEXER_PROCESSOR_POSTGRES_CONNECTION_STRING: postgres://postgres:password@postgres:5432/postgres - AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} - AWS_REGION: ${AWS_DEFAULT_REGION} - volumes: - - ${DOT_MOVEMENT_PATH}:/.movement - healthcheck: - test: [ "CMD-SHELL", "echo 'health check'" ] - retries: 30 - interval: 10s - timeout: 5s - - # turn off underlying da light nodes - celestia-light-node: - image: busybox - container_name: celestia-light-node - command: sleep infinity - environment: - - DOT_MOVEMENT_PATH=/.movement - - CELESTIA_RPC_ADDRESS=celestia-light-node:26657 - volumes: - - ${DOT_MOVEMENT_PATH}:/.movement - depends_on: - setup: - condition: service_healthy - healthcheck: - test: [ "CMD-SHELL", "echo 'health check'" ] - retries: 3 - start_period: 3s - restart: on-failure:3 - - # turn off celestia-light-node-synced - celestia-light-node-synced: - image: busybox - container_name: celestia-light-node-synced - command: echo "No sync check when following." - environment: - - DOT_MOVEMENT_PATH=/.movement - volumes: - - ${DOT_MOVEMENT_PATH}:/.movement - depends_on: - celestia-light-node: - condition: service_healthy \ No newline at end of file diff --git a/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml b/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml new file mode 100644 index 000000000..d5b1ba551 --- /dev/null +++ b/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml @@ -0,0 +1,47 @@ +services: + setup: + image: ghcr.io/movementlabsxyz/movement-full-node-setup:${CONTAINER_REV} + container_name: setup + environment: + CELESTIA_NAMESPACE: 0x3073030730 + CELESTIA_NETWORK: mainnet + # ws://celestia-light-node:26658 + CELESTIA_WEBSOCKET_CONNECTION_PROTOCOL: ws + CELESTIA_WEBSOCKET_CONNECTION_HOSTNAME: celestia-light-node + CELESTIA_WEBSOCKET_CONNECTION_PORT: 26658 + INDEXER_PROCESSOR_POSTGRES_CONNECTION_STRING: postgres://postgres:password@postgres:5432/postgres + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_REGION: ${AWS_DEFAULT_REGION} + volumes: + - ${DOT_MOVEMENT_PATH}:/.movement + healthcheck: + test: [ "CMD-SHELL", "echo 'health check'" ] + retries: 30 + interval: 10s + timeout: 5s + + celestia-light-node: + image: ghcr.io/celestiaorg/celestia-node:v0.21.5 + container_name: celestia-light-node + command: | + celestia light start + --core.ip rpc.celestia.pops.one + --p2p.network celestia + --keyring.backend test + --keyring.keyname movement_celestia_light + --log.level WARN + environment: + - NODE_TYPE=light + - P2P_NETWORK=celestia + volumes: + - ${HOME}/.celestia-light/keys:/home/celestia/keys + ports: + - 26658:26658 + depends_on: + setup: + condition: service_healthy + healthcheck: + # FIXME: check the node is truly up on port 26658 + test: [ "CMD-SHELL", "echo 'health check'" ] + restart: on-failure:3 From bc9082592383da3274ed71f049182d43040c4322 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Fri, 21 Feb 2025 17:45:39 +0200 Subject: [PATCH 27/35] fix(docker): workable celestia light node check --- .../movement-full-node/docker-compose.celestia-mainnet.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml b/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml index d5b1ba551..2d6eeb299 100644 --- a/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml +++ b/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml @@ -42,6 +42,5 @@ services: setup: condition: service_healthy healthcheck: - # FIXME: check the node is truly up on port 26658 - test: [ "CMD-SHELL", "echo 'health check'" ] + test: "celestia node info" restart: on-failure:3 From 3e7ff9dc97e8047203e6dbcdabf348799a9ffa36 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Fri, 21 Feb 2025 17:47:43 +0200 Subject: [PATCH 28/35] fix(docker): workable CELESTIA_NAMESPACE A string starting with 0x needs quotes, otherwise it's treated as a number in YAML. --- .../movement-full-node/docker-compose.celestia-mainnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml b/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml index 2d6eeb299..31168b77d 100644 --- a/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml +++ b/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml @@ -3,7 +3,7 @@ services: image: ghcr.io/movementlabsxyz/movement-full-node-setup:${CONTAINER_REV} container_name: setup environment: - CELESTIA_NAMESPACE: 0x3073030730 + CELESTIA_NAMESPACE: "0xa21de06de7aa38c2a812" CELESTIA_NETWORK: mainnet # ws://celestia-light-node:26658 CELESTIA_WEBSOCKET_CONNECTION_PROTOCOL: ws From 5a07c0e384d1e9b3404a259b26f83d74bb206589 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Fri, 21 Feb 2025 21:20:15 +0200 Subject: [PATCH 29/35] feat: Run celestia light node with --node.store --- Cargo.lock | 5 +- Cargo.toml | 2 +- .../src/celestia_light/arabica.rs | 22 +++++-- .../src/celestia_light/mainnet.rs | 19 ++++-- .../src/celestia_light/mocha.rs | 19 ++++-- .../da/movement/protocol/setup/src/arabica.rs | 34 +++++----- .../protocol/setup/src/common/celestia.rs | 63 +++++++++++++++---- .../da/movement/protocol/setup/src/mainnet.rs | 35 +++++------ .../da/movement/protocol/setup/src/mocha.rs | 35 +++++------ util/commander/Cargo.toml | 7 ++- util/commander/src/lib.rs | 38 ++++++----- 11 files changed, 173 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5916a7e1..436da8c66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6268,6 +6268,7 @@ version = "0.0.2" dependencies = [ "anyhow", "futures", + "itertools 0.13.0", "tokio", "tracing", ] @@ -11696,7 +11697,7 @@ name = "movement-algs" version = "0.0.2" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.13.0", "movement-types", "tokio", ] @@ -11783,7 +11784,7 @@ dependencies = [ "dot-movement", "ed25519-dalek 2.1.1", "futures", - "itertools 0.12.1", + "itertools 0.13.0", "maptos-execution-util", "mcr-settlement-client", "movement-config", diff --git a/Cargo.toml b/Cargo.toml index 7a4a737a2..7f6f821f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -288,7 +288,7 @@ hex = { version = "0.4.3", default-features = false, features = [ ics23 = { version = "0.11.0" } k256 = { version = "0.13.3" } keccak-hash = "0.10.0" -itertools = { version = "0.12.1", default-features = false } +itertools = { version = "0.13.0", default-features = false } jmt = "0.9.0" log = "0.4.21" mirai-annotations = "1.10.1" diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs index 664b150bf..75db9b4ce 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs @@ -1,3 +1,6 @@ +use std::ffi::OsStr; +use std::iter; + #[derive(Debug, Clone)] pub struct Arabica; @@ -8,15 +11,22 @@ impl Arabica { pub async fn run( &self, - _dot_movement: dot_movement::DotMovement, + dot_movement: dot_movement::DotMovement, config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { - // celestia light start --core.ip validator-1.celestia-arabica-11.com --p2p.network arabica + let node_store_dir = dot_movement + .get_path() + .join("celestia") + .join(&config.appd.celestia_chain_id) + .join(".celestia-light"); + commander::run_command( "celestia", - &[ + [ "light", "start", + "--keyring.backend", + "test", "--keyring.keyname", &config.light.key_name, "--core.ip", @@ -25,7 +35,11 @@ impl Arabica { "arabica", "--log.level", "FATAL", - ], + "--node.store", + ] + .iter() + .map(AsRef::::as_ref) + .chain(iter::once(node_store_dir.as_ref())), ) .await?; diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs index 205b6bcf5..46df43f27 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs @@ -1,3 +1,5 @@ +use std::{ffi::OsStr, iter}; + #[derive(Debug, Clone)] pub struct Mainnet; @@ -8,13 +10,18 @@ impl Mainnet { pub async fn run( &self, - _dot_movement: dot_movement::DotMovement, + dot_movement: dot_movement::DotMovement, config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { - // celestia light start --core.ip rpc.celestia.pops.one --p2p.network celestia + let node_store_dir = dot_movement + .get_path() + .join("celestia") + .join(&config.appd.celestia_chain_id) + .join(".celestia-light"); + commander::run_command( "celestia", - &[ + [ "light", "start", "--keyring.backend", @@ -27,7 +34,11 @@ impl Mainnet { "celestia", "--log.level", "FATAL", - ], + "--node.store", + ] + .iter() + .map(AsRef::::as_ref) + .chain(iter::once(node_store_dir.as_ref())), ) .await?; diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs index 6cf10c5ce..a2297dfad 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs @@ -1,3 +1,5 @@ +use std::{ffi::OsStr, iter}; + #[derive(Debug, Clone)] pub struct Mocha; @@ -8,13 +10,18 @@ impl Mocha { pub async fn run( &self, - _dot_movement: dot_movement::DotMovement, + dot_movement: dot_movement::DotMovement, config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { - // celestia light start --core.ip validator-1.celestia-mocha-11.com --p2p.network mocha + let node_store_dir = dot_movement + .get_path() + .join("celestia") + .join(&config.appd.celestia_chain_id) + .join(".celestia-light"); + commander::run_command( "celestia", - &[ + [ "light", "start", "--keyring.backend", @@ -27,7 +34,11 @@ impl Mocha { "mocha", "--log.level", "FATAL", - ], + "--node.store", + ] + .iter() + .map(AsRef::::as_ref) + .chain(iter::once(node_store_dir.as_ref())), ) .await?; diff --git a/protocol-units/da/movement/protocol/setup/src/arabica.rs b/protocol-units/da/movement/protocol/setup/src/arabica.rs index f2abf85d2..eb56ca4ee 100644 --- a/protocol-units/da/movement/protocol/setup/src/arabica.rs +++ b/protocol-units/da/movement/protocol/setup/src/arabica.rs @@ -55,30 +55,24 @@ impl Arabica { Ok(address.to_string()) } - pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { - // celestia light init --p2p.network arabica - run_command( - "celestia", - &["light", "init", "--p2p.network", "arabica", "--keyring.backend", "test"], - ) - .await?; - - Ok(()) + async fn celestia_light_init( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result<(), anyhow::Error> { + common::celestia::celestia_light_init(dot_movement, config, "arabica").await } pub async fn get_da_block_height(&self) -> Result { common::celestia::current_block_height("https://rpc.celestia-arabica-11.com").await } - pub async fn get_auth_token(&self) -> Result { - // celestia light auth admin --p2p.network arabica - let auth_token = - run_command("celestia", &["light", "auth", "admin", "--p2p.network", "arabica"]) - .await? - .trim() - .to_string(); - - Ok(auth_token) + async fn get_auth_token( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result { + common::celestia::get_auth_token(dot_movement, config, "arabica").await } pub async fn setup_celestia( @@ -91,14 +85,14 @@ impl Arabica { let mut config = common::celestia::make_dirs(dot_movement.clone(), config).await?; // celestia light init --p2p.network arabica - self.celestia_light_init().await?; + self.celestia_light_init(&dot_movement, &config).await?; // get the arabica 11 address let address = self.get_arabica_11_address().await?; config.appd.celestia_validator_address.replace(address.clone()); // get the auth token - let auth_token = self.get_auth_token().await?; + let auth_token = self.get_auth_token(&dot_movement, &config).await?; config.appd.celestia_auth_token.replace(auth_token.clone()); // get the initial block height diff --git a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs index b0555def5..c3f994c3e 100644 --- a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs +++ b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs @@ -1,11 +1,16 @@ use crate::common; use anyhow::Context; use celestia_types::nmt::Namespace; +use commander::run_command; use dot_movement::DotMovement; use movement_da_util::config::Config; use rand::Rng; use tracing::info; +use std::ffi::OsStr; +use std::iter; +use std::path::PathBuf; + fn random_10_bytes() -> [u8; 10] { let mut rng = rand::thread_rng(); rng.gen() @@ -19,16 +24,16 @@ pub fn random_namespace() -> Namespace { Namespace::new_v0(&random_10_bytes()).unwrap() } +fn celestia_chain_dir(dot_movement: &DotMovement, config: &Config) -> PathBuf { + dot_movement.get_path().join("celestia").join(&config.appd.celestia_chain_id) +} + pub fn initialize_celestia_config( dot_movement: DotMovement, mut config: Config, ) -> Result { - // use the dot movement path to set up the celestia app and node paths - let dot_movement_path = dot_movement.get_path(); - - let celestia_chain_id = if config.celestia_force_new_chain { + if config.celestia_force_new_chain { // if forced just replace the chain id with a random one - config.appd.celestia_chain_id = random_chain_id(); config.appd.celestia_namespace = random_namespace(); config.appd.celestia_chain_id.clone() @@ -39,9 +44,7 @@ pub fn initialize_celestia_config( // update the app path with the chain id config.appd.celestia_path.replace( - dot_movement_path - .join("celestia") - .join(celestia_chain_id.clone()) + celestia_chain_dir(&dot_movement, &config) .join(".celestia-app") .to_str() .ok_or(anyhow::anyhow!("Failed to convert path to string."))? @@ -50,9 +53,7 @@ pub fn initialize_celestia_config( // update the node path with the chain id config.bridge.celestia_bridge_path.replace( - dot_movement_path - .join("celestia") - .join(celestia_chain_id.clone()) + celestia_chain_dir(&dot_movement, &config) .join(".celestia-node") .to_str() .ok_or(anyhow::anyhow!("Failed to convert path to string."))? @@ -62,6 +63,46 @@ pub fn initialize_celestia_config( Ok(config) } +pub async fn celestia_light_init( + dot_movement: &DotMovement, + config: &Config, + network: &str, +) -> Result<(), anyhow::Error> { + let node_store_dir = celestia_chain_dir(dot_movement, config).join(".celestia-light"); + // celestia light init --p2p.network --keyring.backend test --node_store + run_command( + "celestia", + ["light", "init", "--p2p.network", network, "--keyring.backend", "test", "--node.store"] + .iter() + .map(AsRef::::as_ref) + .chain(iter::once(node_store_dir.as_ref())), + ) + .await?; + + Ok(()) +} + +pub async fn get_auth_token( + dot_movement: &DotMovement, + config: &Config, + network: &str, +) -> Result { + let node_store_dir = celestia_chain_dir(dot_movement, config).join(".celestia-light"); + // celestia light auth admin --p2p.network mocha --node.store + let auth_token = run_command( + "celestia", + ["light", "auth", "admin", "--p2p.network", network, "--node.store"] + .iter() + .map(AsRef::::as_ref) + .chain(iter::once(node_store_dir.as_ref())), + ) + .await? + .trim() + .to_string(); + + Ok(auth_token) +} + pub async fn make_dirs( _dot_movement: DotMovement, config: Config, diff --git a/protocol-units/da/movement/protocol/setup/src/mainnet.rs b/protocol-units/da/movement/protocol/setup/src/mainnet.rs index 50c41c460..86f99e19d 100644 --- a/protocol-units/da/movement/protocol/setup/src/mainnet.rs +++ b/protocol-units/da/movement/protocol/setup/src/mainnet.rs @@ -1,5 +1,4 @@ use crate::common; -use commander::run_command; use dot_movement::DotMovement; use movement_da_util::config::Config; use tracing::info; @@ -12,30 +11,24 @@ impl Mainnet { Self } - pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { - // celestia light init --p2p.network celestia --keyring.backend test - run_command( - "celestia", - &["light", "init", "--p2p.network", "celestia", "--keyring.backend", "test"], - ) - .await?; - - Ok(()) + async fn celestia_light_init( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result<(), anyhow::Error> { + common::celestia::celestia_light_init(dot_movement, config, "celestia").await } pub async fn get_da_block_height(&self) -> Result { common::celestia::current_block_height("https://rpc.celestia.pops.one").await } - pub async fn get_auth_token(&self) -> Result { - // celestia light auth admin --p2p.network celestia - let auth_token = - run_command("celestia", &["light", "auth", "admin", "--p2p.network", "celestia"]) - .await? - .trim() - .to_string(); - - Ok(auth_token) + async fn get_auth_token( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result { + common::celestia::get_auth_token(dot_movement, config, "celestia").await } pub async fn setup_celestia( @@ -48,10 +41,10 @@ impl Mainnet { let mut config = common::celestia::make_dirs(dot_movement.clone(), config).await?; // celestia light init --p2p.network celestia - self.celestia_light_init().await?; + self.celestia_light_init(&dot_movement, &config).await?; // get the auth token - let auth_token = self.get_auth_token().await?; + let auth_token = self.get_auth_token(&dot_movement, &config).await?; config.appd.celestia_auth_token.replace(auth_token.clone()); // get the initial block height diff --git a/protocol-units/da/movement/protocol/setup/src/mocha.rs b/protocol-units/da/movement/protocol/setup/src/mocha.rs index d0672c382..e33418019 100644 --- a/protocol-units/da/movement/protocol/setup/src/mocha.rs +++ b/protocol-units/da/movement/protocol/setup/src/mocha.rs @@ -1,5 +1,4 @@ use crate::common; -use commander::run_command; use dot_movement::DotMovement; use movement_da_util::config::Config; use tracing::info; @@ -12,30 +11,24 @@ impl Mocha { Self } - pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { - // celestia light init --p2p.network mocha - run_command( - "celestia", - &["light", "init", "--p2p.network", "mocha", "--keyring.backend", "test"], - ) - .await?; - - Ok(()) + async fn celestia_light_init( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result<(), anyhow::Error> { + common::celestia::celestia_light_init(dot_movement, config, "mocha").await } pub async fn get_da_block_height(&self) -> Result { common::celestia::current_block_height("https://rpc-mocha.pops.one").await } - pub async fn get_auth_token(&self) -> Result { - // celestia light auth admin --p2p.network mocha - let auth_token = - run_command("celestia", &["light", "auth", "admin", "--p2p.network", "mocha"]) - .await? - .trim() - .to_string(); - - Ok(auth_token) + async fn get_auth_token( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result { + common::celestia::get_auth_token(dot_movement, config, "mocha").await } pub async fn setup_celestia( @@ -48,10 +41,10 @@ impl Mocha { let mut config = common::celestia::make_dirs(dot_movement.clone(), config).await?; // celestia light init --p2p.network mocha - self.celestia_light_init().await?; + self.celestia_light_init(&dot_movement, &config).await?; // get the auth token - let auth_token = self.get_auth_token().await?; + let auth_token = self.get_auth_token(&dot_movement, &config).await?; config.appd.celestia_auth_token.replace(auth_token.clone()); // get the initial block height diff --git a/util/commander/Cargo.toml b/util/commander/Cargo.toml index e78460920..ded91b392 100644 --- a/util/commander/Cargo.toml +++ b/util/commander/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "commander" version = { workspace = true } -edition = { workspace = true } -license = { workspace = true } +edition = { workspace = true } +license = { workspace = true } authors = { workspace = true } repository = { workspace = true } homepage = { workspace = true } @@ -16,6 +16,7 @@ anyhow = { workspace = true } tokio = { workspace = true } futures = { workspace = true } tracing = { workspace = true } +itertools = { workspace = true } [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/util/commander/src/lib.rs b/util/commander/src/lib.rs index b5a16aaf4..00c96243b 100644 --- a/util/commander/src/lib.rs +++ b/util/commander/src/lib.rs @@ -1,10 +1,14 @@ use anyhow::Result; use futures::future::try_join; -use std::process::Stdio; +use itertools::Itertools; use tokio::io::{self, AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::process::Command; use tokio::signal::unix::{signal, SignalKind}; use tokio::task::JoinHandle; +use tracing::info; + +use std::ffi::OsStr; +use std::process::Stdio; async fn pipe_output( reader: R, @@ -37,9 +41,19 @@ async fn pipe_error_output( } /// Runs a command, piping its output to stdout and stderr, and returns the stdout output if successful. -pub async fn run_command(command: &str, args: &[&str]) -> Result { - // print command out with args joined by space - tracing::info!("Running command: {} {}", command, args.join(" ")); +pub async fn run_command(command: C, args: I) -> Result +where + C: AsRef, + I: IntoIterator, + S: AsRef, +{ + let mut command = Command::new(command); + command.args(args); + + let cmd_display = command.as_std().get_program().to_string_lossy().into_owned(); + let args_display = command.as_std().get_args().map(|s| s.to_string_lossy()).join(" "); + + info!("Running command: {cmd_display} {args_display}"); // Setup signal handling to terminate the child process let (tx, rx) = tokio::sync::oneshot::channel(); @@ -62,17 +76,13 @@ pub async fn run_command(command: &str, args: &[&str]) -> Result { } }); - let mut child = Command::new(command) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; + let mut child = command.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?; let stdout = child.stdout.take().ok_or_else(|| { - anyhow::anyhow!("Failed to capture standard output from command {}", command) + anyhow::anyhow!("Failed to capture standard output from command {cmd_display}") })?; let stderr = child.stderr.take().ok_or_else(|| { - anyhow::anyhow!("Failed to capture standard error from command {}", command) + anyhow::anyhow!("Failed to capture standard error from command {cmd_display}") })?; let mut stdout_output = String::new(); @@ -92,16 +102,14 @@ pub async fn run_command(command: &str, args: &[&str]) -> Result { } _ = rx => { let _ = child.kill().await; - return Err(anyhow::anyhow!("Command {} was terminated by signal", command)); + return Err(anyhow::anyhow!("Command {cmd_display} was terminated by signal")); } } let status = child.wait().await?; if !status.success() { return Err(anyhow::anyhow!( - "Command {} failed with args {:?}\nError Output: {}", - command, - args, + "Command {cmd_display} failed with args {args_display}\nError Output: {}", stderr_output )); } From 796fc4c9af1207498679e2124fbb356cdb1e4249 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Fri, 21 Feb 2025 21:37:02 +0200 Subject: [PATCH 30/35] fix(docker): use --node.store with celestia light node --- .../movement-full-node/docker-compose.celestia-mainnet.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml b/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml index 31168b77d..deaea05dd 100644 --- a/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml +++ b/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml @@ -28,6 +28,7 @@ services: celestia light start --core.ip rpc.celestia.pops.one --p2p.network celestia + --node.store /.movement/celestia/movement/.celestia-light --keyring.backend test --keyring.keyname movement_celestia_light --log.level WARN @@ -35,7 +36,7 @@ services: - NODE_TYPE=light - P2P_NETWORK=celestia volumes: - - ${HOME}/.celestia-light/keys:/home/celestia/keys + - ${DOT_MOVEMENT_PATH}/celestia:/.movement/celestia ports: - 26658:26658 depends_on: From 742e8840769b494a502ddcea59ccb0a52465e02f Mon Sep 17 00:00:00 2001 From: Liam Monninger Date: Sat, 22 Feb 2025 12:01:40 -0800 Subject: [PATCH 31/35] fix: lumina version. --- Cargo.lock | 183 +++++++----------- Cargo.toml | 4 +- .../movement/providers/celestia/src/da/mod.rs | 4 +- 3 files changed, 77 insertions(+), 114 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5916a7e1..296692db4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5286,9 +5286,9 @@ dependencies = [ [[package]] name = "blockstore" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7679095248a6dc7555fae81154ed1baef264383c16621ef881a219576c72a9be" +checksum = "0a8962daed8fb337472d9c4215006443acba1e40c6c91c9d4a3f440d1fb30436" dependencies = [ "cid", "dashmap 6.1.0", @@ -5856,91 +5856,45 @@ dependencies = [ [[package]] name = "celestia-proto" -version = "0.5.0" -source = "git+https://github.com/movementlabsxyz/lumina?rev=2d16e6733949f6bf70849eb60c14114e6d8ea63e#2d16e6733949f6bf70849eb60c14114e6d8ea63e" +version = "0.7.0" +source = "git+https://github.com/movementlabsxyz/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" dependencies = [ - "celestia-tendermint-proto", + "bytes 1.8.0", "prost 0.13.3", - "prost-build 0.12.6", + "prost-build", "prost-types 0.13.3", "protox", "serde", + "subtle-encoding", + "tendermint-proto", ] [[package]] name = "celestia-rpc" -version = "0.7.1" -source = "git+https://github.com/movementlabsxyz/lumina?rev=2d16e6733949f6bf70849eb60c14114e6d8ea63e#2d16e6733949f6bf70849eb60c14114e6d8ea63e" +version = "0.9.0" +source = "git+https://github.com/movementlabsxyz/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" dependencies = [ "async-trait", "celestia-types", - "futures", "http 1.1.0", "jsonrpsee", + "prost 0.13.3", "serde", "thiserror 1.0.69", "tracing", ] -[[package]] -name = "celestia-tendermint" -version = "0.32.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce8c92a01145f79a0f3ac7c44a43a9b5ee58e8a4c716b56d98833a3848db1afd" -dependencies = [ - "bytes 1.8.0", - "celestia-tendermint-proto", - "digest 0.10.7", - "ed25519 2.2.3", - "ed25519-consensus", - "flex-error", - "futures", - "num-traits", - "once_cell", - "prost 0.12.6", - "prost-types 0.12.6", - "serde", - "serde_bytes", - "serde_json", - "serde_repr", - "sha2 0.10.8", - "signature 2.2.0", - "subtle", - "subtle-encoding", - "time", - "zeroize", -] - -[[package]] -name = "celestia-tendermint-proto" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bbd5c2345d29ea85566ad8a1114cc0dfb20dacf8f9e0722d64886d0e1e064f7" -dependencies = [ - "bytes 1.8.0", - "flex-error", - "num-derive", - "num-traits", - "prost 0.13.3", - "prost-types 0.13.3", - "serde", - "serde_bytes", - "subtle-encoding", - "time", -] - [[package]] name = "celestia-types" -version = "0.8.0" -source = "git+https://github.com/movementlabsxyz/lumina?rev=2d16e6733949f6bf70849eb60c14114e6d8ea63e#2d16e6733949f6bf70849eb60c14114e6d8ea63e" +version = "0.10.0" +source = "git+https://github.com/movementlabsxyz/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" dependencies = [ "base64 0.22.1", "bech32", + "bitvec 1.0.1", "blockstore", "bytes 1.8.0", "celestia-proto", - "celestia-tendermint", - "celestia-tendermint-proto", "cid", "const_format", "enum_dispatch", @@ -5954,6 +5908,8 @@ dependencies = [ "serde", "serde_repr", "sha2 0.10.8", + "tendermint", + "tendermint-proto", "thiserror 1.0.69", "time", ] @@ -7720,16 +7676,6 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" -[[package]] -name = "eyre" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" -dependencies = [ - "indenter", - "once_cell", -] - [[package]] name = "fail" version = "0.5.1" @@ -7934,7 +7880,6 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c606d892c9de11507fa0dcffc116434f94e105d0bbdc4e405b61519464c49d7b" dependencies = [ - "eyre", "paste", ] @@ -9497,12 +9442,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "indenter" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" - [[package]] name = "indexmap" version = "1.9.3" @@ -14160,27 +14099,6 @@ dependencies = [ "prost-derive 0.13.3", ] -[[package]] -name = "prost-build" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" -dependencies = [ - "bytes 1.8.0", - "heck 0.5.0", - "itertools 0.12.1", - "log", - "multimap", - "once_cell", - "petgraph 0.6.5", - "prettyplease", - "prost 0.12.6", - "prost-types 0.12.6", - "regex", - "syn 2.0.87", - "tempfile", -] - [[package]] name = "prost-build" version = "0.13.3" @@ -14243,15 +14161,15 @@ dependencies = [ [[package]] name = "prost-reflect" -version = "0.13.1" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f5eec97d5d34bdd17ad2db2219aabf46b054c6c41bd5529767c9ce55be5898f" +checksum = "a7b318f733603136dcc61aa9e77c928d67f87d2436c34ec052ba3f1b5ca219de" dependencies = [ "logos", "miette", "once_cell", - "prost 0.12.6", - "prost-types 0.12.6", + "prost 0.13.3", + "prost-types 0.13.3", ] [[package]] @@ -14308,28 +14226,28 @@ dependencies = [ [[package]] name = "protox" -version = "0.6.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac532509cee918d40f38c3e12f8ef9230f215f017d54de7dd975015538a42ce7" +checksum = "6f352af331bf637b8ecc720f7c87bf903d2571fa2e14a66e9b2558846864b54a" dependencies = [ "bytes 1.8.0", "miette", - "prost 0.12.6", + "prost 0.13.3", "prost-reflect", - "prost-types 0.12.6", + "prost-types 0.13.3", "protox-parse", "thiserror 1.0.69", ] [[package]] name = "protox-parse" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6c33f43516fe397e2f930779d720ca12cd057f7da4cd6326a0ef78d69dee96" +checksum = "a3a462d115462c080ae000c29a47f0b3985737e5d3a995fcdbcaa5c782068dde" dependencies = [ "logos", "miette", - "prost-types 0.12.6", + "prost-types 0.13.3", "thiserror 1.0.69", ] @@ -16759,6 +16677,51 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "tendermint" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9703e34d940c2a293804752555107f8dbe2b84ec4c6dd5203831235868105d2" +dependencies = [ + "bytes 1.8.0", + "digest 0.10.7", + "ed25519 2.2.3", + "ed25519-consensus", + "flex-error", + "futures", + "k256", + "num-traits", + "once_cell", + "prost 0.13.3", + "ripemd", + "serde", + "serde_bytes", + "serde_json", + "serde_repr", + "sha2 0.10.8", + "signature 2.2.0", + "subtle", + "subtle-encoding", + "tendermint-proto", + "time", + "zeroize", +] + +[[package]] +name = "tendermint-proto" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9e1705aa0fa5ecb2c6aa7fb78c2313c4a31158ea5f02048bf318f849352eb" +dependencies = [ + "bytes 1.8.0", + "flex-error", + "prost 0.13.3", + "serde", + "serde_bytes", + "subtle-encoding", + "time", +] + [[package]] name = "tera" version = "1.20.0" @@ -17342,7 +17305,7 @@ checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" dependencies = [ "prettyplease", "proc-macro2", - "prost-build 0.13.3", + "prost-build", "prost-types 0.13.3", "quote", "syn 2.0.87", diff --git a/Cargo.toml b/Cargo.toml index 7a4a737a2..c9ea23c6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -228,8 +228,8 @@ secp256k1 = { version = "0.27", default-features = false, features = [ ] } ## Celestia Dependencies -celestia-rpc = { git = "https://github.com/movementlabsxyz/lumina", rev = "2d16e6733949f6bf70849eb60c14114e6d8ea63e" } #{ version = "0.7.0" } -celestia-types = { git = "https://github.com/movementlabsxyz/lumina", rev = "2d16e6733949f6bf70849eb60c14114e6d8ea63e" } #{ version = "0.7.0" } +celestia-rpc = { git = "https://github.com/movementlabsxyz/lumina", rev = "c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5" } #{ version = "0.7.0" } +celestia-types = { git = "https://github.com/movementlabsxyz/lumina", rev = "c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5" } #{ version = "0.7.0" } # External Dependencies diff --git a/protocol-units/da/movement/providers/celestia/src/da/mod.rs b/protocol-units/da/movement/providers/celestia/src/da/mod.rs index 84135c6e6..2ff6300d1 100644 --- a/protocol-units/da/movement/providers/celestia/src/da/mod.rs +++ b/protocol-units/da/movement/providers/celestia/src/da/mod.rs @@ -1,6 +1,6 @@ use crate::blob::ir::{into_da_blob, CelestiaDaBlob}; -use celestia_rpc::{BlobClient, Client, HeaderClient}; -use celestia_types::{nmt::Namespace, Blob as CelestiaBlob, TxConfig}; +use celestia_rpc::{BlobClient, Client, HeaderClient, TxConfig}; +use celestia_types::{nmt::Namespace, Blob as CelestiaBlob}; use movement_da_light_node_da::{Certificate, CertificateStream, DaError, DaOperations}; use movement_da_util::blob::ir::blob::DaBlob; use movement_signer::cryptography::Curve; From 4064b6b70c46e0ad46020f1c763c9dbc4255644e Mon Sep 17 00:00:00 2001 From: Liam Monninger Date: Sun, 23 Feb 2025 09:12:54 -0800 Subject: [PATCH 32/35] fix: debug add blogb logging. --- .../da/movement/providers/celestia/src/da/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/protocol-units/da/movement/providers/celestia/src/da/mod.rs b/protocol-units/da/movement/providers/celestia/src/da/mod.rs index 2ff6300d1..43cc3b69b 100644 --- a/protocol-units/da/movement/providers/celestia/src/da/mod.rs +++ b/protocol-units/da/movement/providers/celestia/src/da/mod.rs @@ -5,10 +5,11 @@ use movement_da_light_node_da::{Certificate, CertificateStream, DaError, DaOpera use movement_da_util::blob::ir::blob::DaBlob; use movement_signer::cryptography::Curve; use serde::{Deserialize, Serialize}; +use std::fmt::Debug; use std::future::Future; use std::pin::Pin; use std::sync::Arc; -use tracing::error; +use tracing::{debug, error}; #[derive(Clone)] pub struct Da @@ -53,18 +54,22 @@ where impl DaOperations for Da where - C: Curve + Send + Sync + Clone + Serialize + for<'de> Deserialize<'de> + 'static, + C: Curve + Send + Sync + Clone + Serialize + Debug + for<'de> Deserialize<'de> + 'static, { fn submit_blob( &self, data: DaBlob, ) -> Pin> + Send + '_>> { Box::pin(async move { + debug!("submitting blob to celestia {:?}", data); + // create the blob let celestia_blob = self .create_new_celestia_blob(data) .map_err(|e| DaError::Internal("failed to create celestia blob".to_string()))?; + debug!("created celestia blob {:?}", celestia_blob); + // submit the blob to the celestia node self.submit_celestia_blob(celestia_blob) .await From 209bf3aee1248ebd6997f75ac130cf4f0a897c5f Mon Sep 17 00:00:00 2001 From: Liam Monninger Date: Sun, 23 Feb 2025 13:46:20 -0800 Subject: [PATCH 33/35] fix: additional logging for da blob. --- .../movement-config/src/releases/biarritz_rc1/da_db.rs | 5 ++++- .../movement-full-node/src/node/tasks/execute_settle.rs | 2 +- protocol-units/da/movement/providers/celestia/src/da/mod.rs | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/networks/movement/movement-config/src/releases/biarritz_rc1/da_db.rs b/networks/movement/movement-config/src/releases/biarritz_rc1/da_db.rs index 07267a8c6..e155a9508 100644 --- a/networks/movement/movement-config/src/releases/biarritz_rc1/da_db.rs +++ b/networks/movement/movement-config/src/releases/biarritz_rc1/da_db.rs @@ -5,12 +5,15 @@ use serde::{Deserialize, Serialize}; pub struct Config { #[serde(default = "default_da_db_path")] pub da_db_path: String, + #[serde(default = "default_start_sync_height")] + pub start_sync_height: u64, } impl Default for Config { fn default() -> Self { - Self { da_db_path: default_da_db_path() } + Self { da_db_path: default_da_db_path(), start_sync_height: default_start_sync_height() } } } env_default!(default_da_db_path, "SUZUKA_DA_DB_PATH", String, "movement-da-db".to_string()); +env_default!(default_start_sync_height, "MOVEMENT_START_SYNC_HEIGHT", u64, 0); diff --git a/networks/movement/movement-full-node/src/node/tasks/execute_settle.rs b/networks/movement/movement-full-node/src/node/tasks/execute_settle.rs index 974a82d39..b2bbddc14 100644 --- a/networks/movement/movement-full-node/src/node/tasks/execute_settle.rs +++ b/networks/movement/movement-full-node/src/node/tasks/execute_settle.rs @@ -69,7 +69,7 @@ where { pub async fn run(mut self) -> anyhow::Result<()> { let synced_height = self.da_db.get_synced_height().await?; - info!("Synced height: {:?}", synced_height); + info!("DA synced height: {:?}", synced_height); let mut blocks_from_da = self .da_light_node_client .stream_read_from_height(StreamReadFromHeightRequest { height: synced_height }) diff --git a/protocol-units/da/movement/providers/celestia/src/da/mod.rs b/protocol-units/da/movement/providers/celestia/src/da/mod.rs index 43cc3b69b..855e51b9a 100644 --- a/protocol-units/da/movement/providers/celestia/src/da/mod.rs +++ b/protocol-units/da/movement/providers/celestia/src/da/mod.rs @@ -84,6 +84,7 @@ where height: u64, ) -> Pin>, DaError>> + Send + '_>> { Box::pin(async move { + debug!("getting blobs at height {height}"); let height = if height == 0 { 1 } else { height }; match self.default_client.blob_get_all(height, &[self.celestia_namespace]).await { @@ -104,6 +105,7 @@ where format!("failed to convert blob: {e}").into(), ) })?; + debug!("got blob {da_blob:?}"); da_blobs.push(da_blob); } From e82cbf6296c401205f414f1ff9a8b8af04d0d740 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Mon, 24 Feb 2025 18:52:21 +0200 Subject: [PATCH 34/35] Config parameter for celestia light node store directory (#1066) --- .../src/celestia_light/arabica.rs | 54 +++--- .../src/celestia_light/mainnet.rs | 51 +++-- .../src/celestia_light/mocha.rs | 53 +++-- .../protocol/setup/src/common/celestia.rs | 49 +++-- .../protocol/util/src/config/default.rs | 5 + .../protocol/util/src/config/light.rs | 13 +- util/commander/src/lib.rs | 183 ++++++++---------- 7 files changed, 188 insertions(+), 220 deletions(-) diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs index 75db9b4ce..a9762eb52 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs @@ -1,5 +1,4 @@ -use std::ffi::OsStr; -use std::iter; +use commander::Command; #[derive(Debug, Clone)] pub struct Arabica; @@ -11,37 +10,30 @@ impl Arabica { pub async fn run( &self, - dot_movement: dot_movement::DotMovement, + _dot_movement: dot_movement::DotMovement, config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { - let node_store_dir = dot_movement - .get_path() - .join("celestia") - .join(&config.appd.celestia_chain_id) - .join(".celestia-light"); - - commander::run_command( - "celestia", - [ - "light", - "start", - "--keyring.backend", - "test", - "--keyring.keyname", - &config.light.key_name, - "--core.ip", - "validator-1.celestia-arabica-11.com", - "--p2p.network", - "arabica", - "--log.level", - "FATAL", - "--node.store", - ] - .iter() - .map(AsRef::::as_ref) - .chain(iter::once(node_store_dir.as_ref())), - ) - .await?; + let mut command = Command::new("celestia"); + command.args([ + "light", + "start", + "--keyring.backend", + "test", + "--keyring.keyname", + &config.light.key_name, + "--core.ip", + "validator-1.celestia-arabica-11.com", + "--p2p.network", + "arabica", + "--log.level", + "FATAL", + ]); + if let Some(path) = &config.light.node_store { + command.arg("--node.store"); + command.arg(path); + } + // FIXME: don't need to capture output + command.run_and_capture_output().await?; Ok(()) } diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs index 46df43f27..6fcd0d6de 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs @@ -1,4 +1,4 @@ -use std::{ffi::OsStr, iter}; +use commander::Command; #[derive(Debug, Clone)] pub struct Mainnet; @@ -10,37 +10,30 @@ impl Mainnet { pub async fn run( &self, - dot_movement: dot_movement::DotMovement, + _dot_movement: dot_movement::DotMovement, config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { - let node_store_dir = dot_movement - .get_path() - .join("celestia") - .join(&config.appd.celestia_chain_id) - .join(".celestia-light"); - - commander::run_command( + let mut command = Command::new("celestia"); + command.args([ + "light", + "start", + "--keyring.backend", + "test", + "--keyring.keyname", + &config.light.key_name, + "--core.ip", + "rpc.celestia.pops.one", + "--p2p.network", "celestia", - [ - "light", - "start", - "--keyring.backend", - "test", - "--keyring.keyname", - &config.light.key_name, - "--core.ip", - "rpc.celestia.pops.one", - "--p2p.network", - "celestia", - "--log.level", - "FATAL", - "--node.store", - ] - .iter() - .map(AsRef::::as_ref) - .chain(iter::once(node_store_dir.as_ref())), - ) - .await?; + "--log.level", + "FATAL", + ]); + if let Some(path) = &config.light.node_store { + command.arg("--node.store"); + command.arg(path); + } + // FIXME: don't need to capture output + command.run_and_capture_output().await?; Ok(()) } diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs index a2297dfad..38deb4ed3 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs @@ -1,4 +1,4 @@ -use std::{ffi::OsStr, iter}; +use commander::Command; #[derive(Debug, Clone)] pub struct Mocha; @@ -10,37 +10,30 @@ impl Mocha { pub async fn run( &self, - dot_movement: dot_movement::DotMovement, + _dot_movement: dot_movement::DotMovement, config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { - let node_store_dir = dot_movement - .get_path() - .join("celestia") - .join(&config.appd.celestia_chain_id) - .join(".celestia-light"); - - commander::run_command( - "celestia", - [ - "light", - "start", - "--keyring.backend", - "test", - "--keyring.keyname", - &config.light.key_name, - "--core.ip", - "rpc-mocha.pops.one", - "--p2p.network", - "mocha", - "--log.level", - "FATAL", - "--node.store", - ] - .iter() - .map(AsRef::::as_ref) - .chain(iter::once(node_store_dir.as_ref())), - ) - .await?; + let mut command = Command::new("celestia"); + command.args([ + "light", + "start", + "--keyring.backend", + "test", + "--keyring.keyname", + &config.light.key_name, + "--core.ip", + "rpc-mocha.pops.one", + "--p2p.network", + "mocha", + "--log.level", + "FATAL", + ]); + if let Some(path) = &config.light.node_store { + command.arg("--node.store"); + command.arg(path); + } + // FIXME: don't need to capture output + command.run_and_capture_output().await?; Ok(()) } diff --git a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs index c3f994c3e..7828d8b91 100644 --- a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs +++ b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs @@ -1,14 +1,12 @@ use crate::common; use anyhow::Context; use celestia_types::nmt::Namespace; -use commander::run_command; +use commander::Command; use dot_movement::DotMovement; use movement_da_util::config::Config; use rand::Rng; use tracing::info; -use std::ffi::OsStr; -use std::iter; use std::path::PathBuf; fn random_10_bytes() -> [u8; 10] { @@ -42,6 +40,12 @@ pub fn initialize_celestia_config( config.appd.celestia_chain_id.clone() }; + // set the node store directory accordingly to the chain id + config + .light + .node_store + .replace(celestia_chain_dir(&dot_movement, &config).join(".celestia-light")); + // update the app path with the chain id config.appd.celestia_path.replace( celestia_chain_dir(&dot_movement, &config) @@ -64,41 +68,36 @@ pub fn initialize_celestia_config( } pub async fn celestia_light_init( - dot_movement: &DotMovement, + _dot_movement: &DotMovement, config: &Config, network: &str, ) -> Result<(), anyhow::Error> { - let node_store_dir = celestia_chain_dir(dot_movement, config).join(".celestia-light"); // celestia light init --p2p.network --keyring.backend test --node_store - run_command( - "celestia", - ["light", "init", "--p2p.network", network, "--keyring.backend", "test", "--node.store"] - .iter() - .map(AsRef::::as_ref) - .chain(iter::once(node_store_dir.as_ref())), - ) - .await?; + let mut command = Command::new("celestia"); + command.args(["light", "init", "--p2p.network", network, "--keyring.backend", "test"]); + if let Some(path) = &config.light.node_store { + command.arg("--node.store"); + command.arg(path); + } + // FIXME: the output does not need to be captured + command.run_and_capture_output().await?; Ok(()) } pub async fn get_auth_token( - dot_movement: &DotMovement, + _dot_movement: &DotMovement, config: &Config, network: &str, ) -> Result { - let node_store_dir = celestia_chain_dir(dot_movement, config).join(".celestia-light"); // celestia light auth admin --p2p.network mocha --node.store - let auth_token = run_command( - "celestia", - ["light", "auth", "admin", "--p2p.network", network, "--node.store"] - .iter() - .map(AsRef::::as_ref) - .chain(iter::once(node_store_dir.as_ref())), - ) - .await? - .trim() - .to_string(); + let mut command = Command::new("celestia"); + command.args(["light", "auth", "admin", "--p2p.network", network]); + if let Some(path) = &config.light.node_store { + command.arg("--node.store"); + command.arg(path); + } + let auth_token = command.run_and_capture_output().await?.trim().to_string(); Ok(auth_token) } diff --git a/protocol-units/da/movement/protocol/util/src/config/default.rs b/protocol-units/da/movement/protocol/util/src/config/default.rs index 123d2d0ee..6c661852a 100644 --- a/protocol-units/da/movement/protocol/util/src/config/default.rs +++ b/protocol-units/da/movement/protocol/util/src/config/default.rs @@ -3,6 +3,7 @@ use celestia_types::nmt::Namespace; use godfig::env_default; use std::env; +use std::path::PathBuf; // The default hostname for the Celestia RPC env_default!( @@ -182,3 +183,7 @@ env_default!( String, "movement_celestia_light".into() ); + +pub fn default_celestia_light_node_store() -> Option { + std::env::var_os("CELESTIA_LIGHT_NODE_STORE").map(Into::into) +} diff --git a/protocol-units/da/movement/protocol/util/src/config/light.rs b/protocol-units/da/movement/protocol/util/src/config/light.rs index 0d4d48da0..da9bc2a74 100644 --- a/protocol-units/da/movement/protocol/util/src/config/light.rs +++ b/protocol-units/da/movement/protocol/util/src/config/light.rs @@ -1,15 +1,24 @@ -use super::default::default_celestia_light_node_key_name; +use super::default::{default_celestia_light_node_key_name, default_celestia_light_node_store}; use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct Config { /// Name of the node's signing key in the keyring + #[serde(default = "default_celestia_light_node_key_name")] pub key_name: String, + /// Path name of the node store directory + #[serde(default)] + pub node_store: Option, } impl Default for Config { fn default() -> Self { - Self { key_name: default_celestia_light_node_key_name() } + Self { + key_name: default_celestia_light_node_key_name(), + node_store: default_celestia_light_node_store(), + } } } diff --git a/util/commander/src/lib.rs b/util/commander/src/lib.rs index 00c96243b..885359bc8 100644 --- a/util/commander/src/lib.rs +++ b/util/commander/src/lib.rs @@ -2,9 +2,8 @@ use anyhow::Result; use futures::future::try_join; use itertools::Itertools; use tokio::io::{self, AsyncBufReadExt, AsyncWriteExt, BufReader}; -use tokio::process::Command; +use tokio::process::Command as InnerCommand; use tokio::signal::unix::{signal, SignalKind}; -use tokio::task::JoinHandle; use tracing::info; use std::ffi::OsStr; @@ -49,124 +48,102 @@ where { let mut command = Command::new(command); command.args(args); + command.run_and_capture_output().await +} - let cmd_display = command.as_std().get_program().to_string_lossy().into_owned(); - let args_display = command.as_std().get_args().map(|s| s.to_string_lossy()).join(" "); +/// Builder for running commands +pub struct Command(InnerCommand); - info!("Running command: {cmd_display} {args_display}"); +impl Command { + pub fn new(program: impl AsRef) -> Self { + let inner = InnerCommand::new(program); + Self(inner) + } - // Setup signal handling to terminate the child process - let (tx, rx) = tokio::sync::oneshot::channel(); + pub fn arg(&mut self, arg: S) -> &mut Self + where + S: AsRef, + { + self.0.arg(arg); + self + } - let mut sigterm = signal(SignalKind::terminate())?; - let mut sigint = signal(SignalKind::interrupt())?; - let mut sigquit = signal(SignalKind::quit())?; + pub fn args(&mut self, args: I) -> &mut Self + where + I: IntoIterator, + S: AsRef, + { + self.0.args(args); + self + } - tokio::spawn(async move { - tokio::select! { - _ = sigterm.recv() => { - let _ = tx.send(()); - } - _ = sigint.recv() => { - let _ = tx.send(()); + pub async fn run_and_capture_output(&mut self) -> Result { + let cmd_display = self.0.as_std().get_program().to_string_lossy().into_owned(); + let args_display = self.0.as_std().get_args().map(|s| s.to_string_lossy()).join(" "); + + info!("Running command: {cmd_display} {args_display}"); + + // Setup signal handling to terminate the child process + let (tx, rx) = tokio::sync::oneshot::channel(); + + let mut sigterm = signal(SignalKind::terminate())?; + let mut sigint = signal(SignalKind::interrupt())?; + let mut sigquit = signal(SignalKind::quit())?; + + tokio::spawn(async move { + tokio::select! { + _ = sigterm.recv() => { + let _ = tx.send(()); + } + _ = sigint.recv() => { + let _ = tx.send(()); + } + _ = sigquit.recv() => { + let _ = tx.send(()); + } } - _ = sigquit.recv() => { - let _ = tx.send(()); - } - } - }); + }); - let mut child = command.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?; + let mut child = self.0.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?; - let stdout = child.stdout.take().ok_or_else(|| { - anyhow::anyhow!("Failed to capture standard output from command {cmd_display}") - })?; - let stderr = child.stderr.take().ok_or_else(|| { - anyhow::anyhow!("Failed to capture standard error from command {cmd_display}") - })?; + let stdout = child.stdout.take().ok_or_else(|| { + anyhow::anyhow!("Failed to capture standard output from command {cmd_display}") + })?; + let stderr = child.stderr.take().ok_or_else(|| { + anyhow::anyhow!("Failed to capture standard error from command {cmd_display}") + })?; - let mut stdout_output = String::new(); - let mut stderr_output = String::new(); + let mut stdout_output = String::new(); + let mut stderr_output = String::new(); - let stdout_writer = io::stdout(); - let stderr_writer = io::stderr(); + let stdout_writer = io::stdout(); + let stderr_writer = io::stderr(); - let stdout_future = pipe_output(stdout, stdout_writer, &mut stdout_output); - let stderr_future = pipe_error_output(stderr, stderr_writer, &mut stderr_output); + let stdout_future = pipe_output(stdout, stdout_writer, &mut stdout_output); + let stderr_future = pipe_error_output(stderr, stderr_writer, &mut stderr_output); - let combined_future = try_join(stdout_future, stderr_future); + let combined_future = try_join(stdout_future, stderr_future); - tokio::select! { - output = combined_future => { - output?; - } - _ = rx => { - let _ = child.kill().await; - return Err(anyhow::anyhow!("Command {cmd_display} was terminated by signal")); - } - } - - let status = child.wait().await?; - if !status.success() { - return Err(anyhow::anyhow!( - "Command {cmd_display} failed with args {args_display}\nError Output: {}", - stderr_output - )); - } - - Ok(stdout_output) -} - -/// Runs a command, piping its output to stdout and stderr, and returns the stdout output if successful. -pub async fn spawn_command( - command: String, - args: Vec, -) -> Result<(Option, JoinHandle>)> { - // print command out with args joined by space - tracing::info!("spawn command: {} {}", command, args.join(" ")); - - let mut child = Command::new(&command) - .args(&args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - - let process_id = child.id(); - let join_handle = tokio::spawn({ - async move { - let stdout = child.stdout.take().ok_or_else(|| { - anyhow::anyhow!("Failed to capture standard output from command {}", command) - })?; - let stderr = child.stderr.take().ok_or_else(|| { - anyhow::anyhow!("Failed to capture standard error from command {}", command) - })?; - - let mut stdout_output = String::new(); - let mut stderr_output = String::new(); - - let stdout_writer = io::stdout(); - let stderr_writer = io::stderr(); - - let stdout_future = pipe_output(stdout, stdout_writer, &mut stdout_output); - let stderr_future = pipe_error_output(stderr, stderr_writer, &mut stderr_output); - - let _ = try_join(stdout_future, stderr_future).await; - - let status = child.wait().await?; - if !status.success() { - return Err(anyhow::anyhow!( - "Command {} spawn failed with args {:?}\nError Output: {}", - command, - args, - stderr_output - )); + tokio::select! { + output = combined_future => { + output?; + } + _ = rx => { + let _ = child.kill().await; + return Err(anyhow::anyhow!("Command {cmd_display} was terminated by signal")); } + } - Ok(stdout_output) + let status = child.wait().await?; + if !status.success() { + return Err(anyhow::anyhow!( + "Command {cmd_display} failed with args {args_display}\nError Output: {}", + stderr_output + )); } - }); - Ok((process_id, join_handle)) + Ok(stdout_output) + } } #[cfg(test)] From c2c40ad7ca31d0c7a94db9f85e6cae8628a96a99 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Tue, 25 Feb 2025 00:19:00 +0200 Subject: [PATCH 35/35] deps: switch lumina back to upstream --- Cargo.lock | 6 +++--- Cargo.toml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65fe64012..559d4c989 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5857,7 +5857,7 @@ dependencies = [ [[package]] name = "celestia-proto" version = "0.7.0" -source = "git+https://github.com/movementlabsxyz/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" +source = "git+https://github.com/eigerco/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" dependencies = [ "bytes 1.8.0", "prost 0.13.3", @@ -5872,7 +5872,7 @@ dependencies = [ [[package]] name = "celestia-rpc" version = "0.9.0" -source = "git+https://github.com/movementlabsxyz/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" +source = "git+https://github.com/eigerco/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" dependencies = [ "async-trait", "celestia-types", @@ -5887,7 +5887,7 @@ dependencies = [ [[package]] name = "celestia-types" version = "0.10.0" -source = "git+https://github.com/movementlabsxyz/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" +source = "git+https://github.com/eigerco/lumina?rev=c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5#c6e5b7f5e3a3040bce4262fe5fba5c21a2637b52" dependencies = [ "base64 0.22.1", "bech32", diff --git a/Cargo.toml b/Cargo.toml index f0ae2a590..8b4eadcc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -228,8 +228,8 @@ secp256k1 = { version = "0.27", default-features = false, features = [ ] } ## Celestia Dependencies -celestia-rpc = { git = "https://github.com/movementlabsxyz/lumina", rev = "c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5" } #{ version = "0.7.0" } -celestia-types = { git = "https://github.com/movementlabsxyz/lumina", rev = "c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5" } #{ version = "0.7.0" } +celestia-rpc = { git = "https://github.com/eigerco/lumina", rev = "c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5" } #{ version = "0.7.0" } +celestia-types = { git = "https://github.com/eigerco/lumina", rev = "c6e5b7f5e3a3040bce4262fe5fba5c21a2637b5" } #{ version = "0.7.0" } # External Dependencies