diff --git a/Cargo.lock b/Cargo.lock index 296692db4..65fe64012 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6224,6 +6224,7 @@ version = "0.0.2" dependencies = [ "anyhow", "futures", + "itertools 0.13.0", "tokio", "tracing", ] @@ -11635,7 +11636,7 @@ name = "movement-algs" version = "0.0.2" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.13.0", "movement-types", "tokio", ] @@ -11722,7 +11723,7 @@ dependencies = [ "dot-movement", "ed25519-dalek 2.1.1", "futures", - "itertools 0.12.1", + "itertools 0.13.0", "maptos-execution-util", "mcr-settlement-client", "movement-config", diff --git a/Cargo.toml b/Cargo.toml index c9ea23c6f..f0ae2a590 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -288,7 +288,7 @@ hex = { version = "0.4.3", default-features = false, features = [ ics23 = { version = "0.11.0" } k256 = { version = "0.13.3" } keccak-hash = "0.10.0" -itertools = { version = "0.12.1", default-features = false } +itertools = { version = "0.13.0", default-features = false } jmt = "0.9.0" log = "0.4.21" mirai-annotations = "1.10.1" diff --git a/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml b/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml index 31168b77d..deaea05dd 100644 --- a/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml +++ b/docker/compose/movement-full-node/docker-compose.celestia-mainnet.yml @@ -28,6 +28,7 @@ services: celestia light start --core.ip rpc.celestia.pops.one --p2p.network celestia + --node.store /.movement/celestia/movement/.celestia-light --keyring.backend test --keyring.keyname movement_celestia_light --log.level WARN @@ -35,7 +36,7 @@ services: - NODE_TYPE=light - P2P_NETWORK=celestia volumes: - - ${HOME}/.celestia-light/keys:/home/celestia/keys + - ${DOT_MOVEMENT_PATH}/celestia:/.movement/celestia ports: - 26658:26658 depends_on: diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs index 664b150bf..75db9b4ce 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/arabica.rs @@ -1,3 +1,6 @@ +use std::ffi::OsStr; +use std::iter; + #[derive(Debug, Clone)] pub struct Arabica; @@ -8,15 +11,22 @@ impl Arabica { pub async fn run( &self, - _dot_movement: dot_movement::DotMovement, + dot_movement: dot_movement::DotMovement, config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { - // celestia light start --core.ip validator-1.celestia-arabica-11.com --p2p.network arabica + let node_store_dir = dot_movement + .get_path() + .join("celestia") + .join(&config.appd.celestia_chain_id) + .join(".celestia-light"); + commander::run_command( "celestia", - &[ + [ "light", "start", + "--keyring.backend", + "test", "--keyring.keyname", &config.light.key_name, "--core.ip", @@ -25,7 +35,11 @@ impl Arabica { "arabica", "--log.level", "FATAL", - ], + "--node.store", + ] + .iter() + .map(AsRef::::as_ref) + .chain(iter::once(node_store_dir.as_ref())), ) .await?; diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs index 205b6bcf5..46df43f27 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mainnet.rs @@ -1,3 +1,5 @@ +use std::{ffi::OsStr, iter}; + #[derive(Debug, Clone)] pub struct Mainnet; @@ -8,13 +10,18 @@ impl Mainnet { pub async fn run( &self, - _dot_movement: dot_movement::DotMovement, + dot_movement: dot_movement::DotMovement, config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { - // celestia light start --core.ip rpc.celestia.pops.one --p2p.network celestia + let node_store_dir = dot_movement + .get_path() + .join("celestia") + .join(&config.appd.celestia_chain_id) + .join(".celestia-light"); + commander::run_command( "celestia", - &[ + [ "light", "start", "--keyring.backend", @@ -27,7 +34,11 @@ impl Mainnet { "celestia", "--log.level", "FATAL", - ], + "--node.store", + ] + .iter() + .map(AsRef::::as_ref) + .chain(iter::once(node_store_dir.as_ref())), ) .await?; diff --git a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs index 6cf10c5ce..a2297dfad 100644 --- a/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs +++ b/protocol-units/da/movement/protocol/celestia-runners/src/celestia_light/mocha.rs @@ -1,3 +1,5 @@ +use std::{ffi::OsStr, iter}; + #[derive(Debug, Clone)] pub struct Mocha; @@ -8,13 +10,18 @@ impl Mocha { pub async fn run( &self, - _dot_movement: dot_movement::DotMovement, + dot_movement: dot_movement::DotMovement, config: movement_da_util::config::Config, ) -> Result<(), anyhow::Error> { - // celestia light start --core.ip validator-1.celestia-mocha-11.com --p2p.network mocha + let node_store_dir = dot_movement + .get_path() + .join("celestia") + .join(&config.appd.celestia_chain_id) + .join(".celestia-light"); + commander::run_command( "celestia", - &[ + [ "light", "start", "--keyring.backend", @@ -27,7 +34,11 @@ impl Mocha { "mocha", "--log.level", "FATAL", - ], + "--node.store", + ] + .iter() + .map(AsRef::::as_ref) + .chain(iter::once(node_store_dir.as_ref())), ) .await?; diff --git a/protocol-units/da/movement/protocol/setup/src/arabica.rs b/protocol-units/da/movement/protocol/setup/src/arabica.rs index f2abf85d2..eb56ca4ee 100644 --- a/protocol-units/da/movement/protocol/setup/src/arabica.rs +++ b/protocol-units/da/movement/protocol/setup/src/arabica.rs @@ -55,30 +55,24 @@ impl Arabica { Ok(address.to_string()) } - pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { - // celestia light init --p2p.network arabica - run_command( - "celestia", - &["light", "init", "--p2p.network", "arabica", "--keyring.backend", "test"], - ) - .await?; - - Ok(()) + async fn celestia_light_init( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result<(), anyhow::Error> { + common::celestia::celestia_light_init(dot_movement, config, "arabica").await } pub async fn get_da_block_height(&self) -> Result { common::celestia::current_block_height("https://rpc.celestia-arabica-11.com").await } - pub async fn get_auth_token(&self) -> Result { - // celestia light auth admin --p2p.network arabica - let auth_token = - run_command("celestia", &["light", "auth", "admin", "--p2p.network", "arabica"]) - .await? - .trim() - .to_string(); - - Ok(auth_token) + async fn get_auth_token( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result { + common::celestia::get_auth_token(dot_movement, config, "arabica").await } pub async fn setup_celestia( @@ -91,14 +85,14 @@ impl Arabica { let mut config = common::celestia::make_dirs(dot_movement.clone(), config).await?; // celestia light init --p2p.network arabica - self.celestia_light_init().await?; + self.celestia_light_init(&dot_movement, &config).await?; // get the arabica 11 address let address = self.get_arabica_11_address().await?; config.appd.celestia_validator_address.replace(address.clone()); // get the auth token - let auth_token = self.get_auth_token().await?; + let auth_token = self.get_auth_token(&dot_movement, &config).await?; config.appd.celestia_auth_token.replace(auth_token.clone()); // get the initial block height diff --git a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs index b0555def5..c3f994c3e 100644 --- a/protocol-units/da/movement/protocol/setup/src/common/celestia.rs +++ b/protocol-units/da/movement/protocol/setup/src/common/celestia.rs @@ -1,11 +1,16 @@ use crate::common; use anyhow::Context; use celestia_types::nmt::Namespace; +use commander::run_command; use dot_movement::DotMovement; use movement_da_util::config::Config; use rand::Rng; use tracing::info; +use std::ffi::OsStr; +use std::iter; +use std::path::PathBuf; + fn random_10_bytes() -> [u8; 10] { let mut rng = rand::thread_rng(); rng.gen() @@ -19,16 +24,16 @@ pub fn random_namespace() -> Namespace { Namespace::new_v0(&random_10_bytes()).unwrap() } +fn celestia_chain_dir(dot_movement: &DotMovement, config: &Config) -> PathBuf { + dot_movement.get_path().join("celestia").join(&config.appd.celestia_chain_id) +} + pub fn initialize_celestia_config( dot_movement: DotMovement, mut config: Config, ) -> Result { - // use the dot movement path to set up the celestia app and node paths - let dot_movement_path = dot_movement.get_path(); - - let celestia_chain_id = if config.celestia_force_new_chain { + if config.celestia_force_new_chain { // if forced just replace the chain id with a random one - config.appd.celestia_chain_id = random_chain_id(); config.appd.celestia_namespace = random_namespace(); config.appd.celestia_chain_id.clone() @@ -39,9 +44,7 @@ pub fn initialize_celestia_config( // update the app path with the chain id config.appd.celestia_path.replace( - dot_movement_path - .join("celestia") - .join(celestia_chain_id.clone()) + celestia_chain_dir(&dot_movement, &config) .join(".celestia-app") .to_str() .ok_or(anyhow::anyhow!("Failed to convert path to string."))? @@ -50,9 +53,7 @@ pub fn initialize_celestia_config( // update the node path with the chain id config.bridge.celestia_bridge_path.replace( - dot_movement_path - .join("celestia") - .join(celestia_chain_id.clone()) + celestia_chain_dir(&dot_movement, &config) .join(".celestia-node") .to_str() .ok_or(anyhow::anyhow!("Failed to convert path to string."))? @@ -62,6 +63,46 @@ pub fn initialize_celestia_config( Ok(config) } +pub async fn celestia_light_init( + dot_movement: &DotMovement, + config: &Config, + network: &str, +) -> Result<(), anyhow::Error> { + let node_store_dir = celestia_chain_dir(dot_movement, config).join(".celestia-light"); + // celestia light init --p2p.network --keyring.backend test --node_store + run_command( + "celestia", + ["light", "init", "--p2p.network", network, "--keyring.backend", "test", "--node.store"] + .iter() + .map(AsRef::::as_ref) + .chain(iter::once(node_store_dir.as_ref())), + ) + .await?; + + Ok(()) +} + +pub async fn get_auth_token( + dot_movement: &DotMovement, + config: &Config, + network: &str, +) -> Result { + let node_store_dir = celestia_chain_dir(dot_movement, config).join(".celestia-light"); + // celestia light auth admin --p2p.network mocha --node.store + let auth_token = run_command( + "celestia", + ["light", "auth", "admin", "--p2p.network", network, "--node.store"] + .iter() + .map(AsRef::::as_ref) + .chain(iter::once(node_store_dir.as_ref())), + ) + .await? + .trim() + .to_string(); + + Ok(auth_token) +} + pub async fn make_dirs( _dot_movement: DotMovement, config: Config, diff --git a/protocol-units/da/movement/protocol/setup/src/mainnet.rs b/protocol-units/da/movement/protocol/setup/src/mainnet.rs index 50c41c460..86f99e19d 100644 --- a/protocol-units/da/movement/protocol/setup/src/mainnet.rs +++ b/protocol-units/da/movement/protocol/setup/src/mainnet.rs @@ -1,5 +1,4 @@ use crate::common; -use commander::run_command; use dot_movement::DotMovement; use movement_da_util::config::Config; use tracing::info; @@ -12,30 +11,24 @@ impl Mainnet { Self } - pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { - // celestia light init --p2p.network celestia --keyring.backend test - run_command( - "celestia", - &["light", "init", "--p2p.network", "celestia", "--keyring.backend", "test"], - ) - .await?; - - Ok(()) + async fn celestia_light_init( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result<(), anyhow::Error> { + common::celestia::celestia_light_init(dot_movement, config, "celestia").await } pub async fn get_da_block_height(&self) -> Result { common::celestia::current_block_height("https://rpc.celestia.pops.one").await } - pub async fn get_auth_token(&self) -> Result { - // celestia light auth admin --p2p.network celestia - let auth_token = - run_command("celestia", &["light", "auth", "admin", "--p2p.network", "celestia"]) - .await? - .trim() - .to_string(); - - Ok(auth_token) + async fn get_auth_token( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result { + common::celestia::get_auth_token(dot_movement, config, "celestia").await } pub async fn setup_celestia( @@ -48,10 +41,10 @@ impl Mainnet { let mut config = common::celestia::make_dirs(dot_movement.clone(), config).await?; // celestia light init --p2p.network celestia - self.celestia_light_init().await?; + self.celestia_light_init(&dot_movement, &config).await?; // get the auth token - let auth_token = self.get_auth_token().await?; + let auth_token = self.get_auth_token(&dot_movement, &config).await?; config.appd.celestia_auth_token.replace(auth_token.clone()); // get the initial block height diff --git a/protocol-units/da/movement/protocol/setup/src/mocha.rs b/protocol-units/da/movement/protocol/setup/src/mocha.rs index d0672c382..e33418019 100644 --- a/protocol-units/da/movement/protocol/setup/src/mocha.rs +++ b/protocol-units/da/movement/protocol/setup/src/mocha.rs @@ -1,5 +1,4 @@ use crate::common; -use commander::run_command; use dot_movement::DotMovement; use movement_da_util::config::Config; use tracing::info; @@ -12,30 +11,24 @@ impl Mocha { Self } - pub async fn celestia_light_init(&self) -> Result<(), anyhow::Error> { - // celestia light init --p2p.network mocha - run_command( - "celestia", - &["light", "init", "--p2p.network", "mocha", "--keyring.backend", "test"], - ) - .await?; - - Ok(()) + async fn celestia_light_init( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result<(), anyhow::Error> { + common::celestia::celestia_light_init(dot_movement, config, "mocha").await } pub async fn get_da_block_height(&self) -> Result { common::celestia::current_block_height("https://rpc-mocha.pops.one").await } - pub async fn get_auth_token(&self) -> Result { - // celestia light auth admin --p2p.network mocha - let auth_token = - run_command("celestia", &["light", "auth", "admin", "--p2p.network", "mocha"]) - .await? - .trim() - .to_string(); - - Ok(auth_token) + async fn get_auth_token( + &self, + dot_movement: &DotMovement, + config: &Config, + ) -> Result { + common::celestia::get_auth_token(dot_movement, config, "mocha").await } pub async fn setup_celestia( @@ -48,10 +41,10 @@ impl Mocha { let mut config = common::celestia::make_dirs(dot_movement.clone(), config).await?; // celestia light init --p2p.network mocha - self.celestia_light_init().await?; + self.celestia_light_init(&dot_movement, &config).await?; // get the auth token - let auth_token = self.get_auth_token().await?; + let auth_token = self.get_auth_token(&dot_movement, &config).await?; config.appd.celestia_auth_token.replace(auth_token.clone()); // get the initial block height diff --git a/util/commander/Cargo.toml b/util/commander/Cargo.toml index e78460920..ded91b392 100644 --- a/util/commander/Cargo.toml +++ b/util/commander/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "commander" version = { workspace = true } -edition = { workspace = true } -license = { workspace = true } +edition = { workspace = true } +license = { workspace = true } authors = { workspace = true } repository = { workspace = true } homepage = { workspace = true } @@ -16,6 +16,7 @@ anyhow = { workspace = true } tokio = { workspace = true } futures = { workspace = true } tracing = { workspace = true } +itertools = { workspace = true } [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/util/commander/src/lib.rs b/util/commander/src/lib.rs index b5a16aaf4..00c96243b 100644 --- a/util/commander/src/lib.rs +++ b/util/commander/src/lib.rs @@ -1,10 +1,14 @@ use anyhow::Result; use futures::future::try_join; -use std::process::Stdio; +use itertools::Itertools; use tokio::io::{self, AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::process::Command; use tokio::signal::unix::{signal, SignalKind}; use tokio::task::JoinHandle; +use tracing::info; + +use std::ffi::OsStr; +use std::process::Stdio; async fn pipe_output( reader: R, @@ -37,9 +41,19 @@ async fn pipe_error_output( } /// Runs a command, piping its output to stdout and stderr, and returns the stdout output if successful. -pub async fn run_command(command: &str, args: &[&str]) -> Result { - // print command out with args joined by space - tracing::info!("Running command: {} {}", command, args.join(" ")); +pub async fn run_command(command: C, args: I) -> Result +where + C: AsRef, + I: IntoIterator, + S: AsRef, +{ + let mut command = Command::new(command); + command.args(args); + + let cmd_display = command.as_std().get_program().to_string_lossy().into_owned(); + let args_display = command.as_std().get_args().map(|s| s.to_string_lossy()).join(" "); + + info!("Running command: {cmd_display} {args_display}"); // Setup signal handling to terminate the child process let (tx, rx) = tokio::sync::oneshot::channel(); @@ -62,17 +76,13 @@ pub async fn run_command(command: &str, args: &[&str]) -> Result { } }); - let mut child = Command::new(command) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; + let mut child = command.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?; let stdout = child.stdout.take().ok_or_else(|| { - anyhow::anyhow!("Failed to capture standard output from command {}", command) + anyhow::anyhow!("Failed to capture standard output from command {cmd_display}") })?; let stderr = child.stderr.take().ok_or_else(|| { - anyhow::anyhow!("Failed to capture standard error from command {}", command) + anyhow::anyhow!("Failed to capture standard error from command {cmd_display}") })?; let mut stdout_output = String::new(); @@ -92,16 +102,14 @@ pub async fn run_command(command: &str, args: &[&str]) -> Result { } _ = rx => { let _ = child.kill().await; - return Err(anyhow::anyhow!("Command {} was terminated by signal", command)); + return Err(anyhow::anyhow!("Command {cmd_display} was terminated by signal")); } } let status = child.wait().await?; if !status.success() { return Err(anyhow::anyhow!( - "Command {} failed with args {:?}\nError Output: {}", - command, - args, + "Command {cmd_display} failed with args {args_display}\nError Output: {}", stderr_output )); }