Public Syncador

Cargo.toml

@@ -10,6 +10,7 @@ members = [
     "protocol-units/da/m1/*",
     "protocol-units/sequencing/memseq/*",
     "protocol-units/mempool/*",
+    "protocol-units/syncing/*",
     "protocol-units/settlement/mcr/client",
     "protocol-units/settlement/mcr/config",
     "protocol-units/settlement/mcr/manager",
@@ -90,6 +91,8 @@ monza-config = { path = "networks/monza/monza-config" }
 flocks = { path = "util/flocks" }
 godfig = { path = "util/godfig" }
 movement-tracing = { path = "util/tracing" }
+syncup = { path = "protocol-units/syncing/syncup" }
+syncador = { path = "util/syncador" }
 
 # Serialization and Deserialization
 borsh = { version = "0.10" } # todo: internalize jmt and bump
@@ -277,6 +280,8 @@ uuid = { version = "1.10.0", features = ["v4"] }
 tar = "0.4.41"
 flate2 = "1.0.31"
 blake-3 = "1.4.0"
+regex = "1.10.6"
+glob = "0.3.1"
 
 # trying to pin diesel
 # diesel = "=2.1.1"

docker/compose/suzuka-full-node/docker-compose.celestia-local.yml → docker/compose/suzuka-full-node/docker-compose.local.yml

@@ -1,4 +1,35 @@
 services:
+  setup:
+    image: ghcr.io/movementlabsxyz/suzuka-full-node-setup:${CONTAINER_REV}
+    container_name: setup
+    environment:
+      DOT_MOVEMENT_PATH: /.movement
+      # By default, all the listener hostnames and ports are reasonable.
+      # However, for container networking, we need to specify the hostnames on the connection.
+      # You can also use host.docker.internal for the host machine as all of these ports should be exposed.
+      CELESTIA_RPC_CONNECTION_HOSTNAME: m1-da-light-node-celestia-appd
+      CELESTIA_WEBSOCKET_CONNECTION_HOSTNAME: m1-da-light-node-celestia-bridge
+      M1_DA_LIGHT_NODE_CONNECTION_HOSTNAME: m1-da-light-node
+      MAPTOS_API_CONNECTION_HOSTNAME: suzuka-full-node
+      FAUCET_API_CONNECTION_HOSTNAME: suzuka-faucet-service
+      ETH_RPC_CONNECTION_HOSTNAME: setup
+      ETH_RPC_CONNECTION_PROTOCOL: http
+      ETH_RPC_CONNECTION_PORT: 8090
+      MAYBE_RUN_LOCAL: "true"
+      MAYBE_DEPLOY_MCR: "true"
+      INDEXER_PROCESSOR_POSTGRES_CONNECTION_STRING: postgres://postgres:password@postgres:5432/postgres
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_REGION: ${AWS_DEFAULT_REGION}
+    volumes:
+      - ${DOT_MOVEMENT_PATH}:/.movement
+    ports:
+      - "8090:8090" # port for anvil, this should be moved out into runner
+    healthcheck:
+      test: [ "CMD-SHELL", "nc -zv 0.0.0.0 8090" ]
+      retries: 10
+      interval: 10s
+      timeout: 5s
 
   m1-da-light-node-celestia-appd:
     image: ghcr.io/movementlabsxyz/m1-da-light-node-celestia-appd:${CONTAINER_REV}

docker/compose/suzuka-full-node/docker-compose.remote-no-celestia-light-node.yml

@@ -0,0 +1,38 @@
+services:
+  setup:
+    image: ghcr.io/movementlabsxyz/suzuka-full-node-setup:${CONTAINER_REV}
+    container_name: setup
+    environment:
+      DOT_MOVEMENT_PATH: /.movement
+      # By default, all the listener hostnames and ports are reasonable.
+      # However, for container networking, we need to specify the hostnames on the connection.
+      # You can also use host.docker.internal for the host machine as all of these ports should be exposed.
+      CELESTIA_RPC_CONNECTION_HOSTNAME: ${CELESTIA_RPC_CONNECTION_HOSTNAME:?CELESTIA_RPC_CONNECTION_HOSTNAME is not set}
+      CELESTIA_RPC_CONNECTION_PORT: ${CELESTIA_RPC_CONNECTION_PORT:?CELESTIA_RPC_CONNECTION_HOSTNAME is not set}
+      CELESTIA_WEBSOCKET_CONNECTION_HOSTNAME: ${CELESTIA_WEBSOCKET_CONNECTION_HOSTNAME:?CELESTIA_WEBSOCKET_CONNECTION_HOSTNAME is not set}
+      CELESTIA_WEBSOCKET_CONNECTION_PORT: ${CELESTIA_WEBSOCKET_CONNECTION_HOSTNAME:?CELESTIA_WEBSOCKET_CONNECTION_HOSTNAME is not set}
+      M1_DA_LIGHT_NODE_CONNECTION_HOSTNAME: m1-da-light-node
+      MAPTOS_API_CONNECTION_HOSTNAME: suzuka-full-node
+      FAUCET_API_CONNECTION_HOSTNAME: suzuka-faucet-service
+      ETH_RPC_CONNECTION_HOSTNAME: ${ETH_RPC_CONNECTION_HOSTNAME:?ETH_RPC_CONNECTION_HOSTNAME is not set}
+      ETH_RPC_CONNECTION_PROTOCOL: ${ETH_RPC_CONNECTION_PROTOCOL:?ETH_RPC_CONNECTION_PROTOCOL is not set}
+      ETH_RPC_CONNECTION_PORT: ${ETH_RPC_CONNECTION_PORT:?ETH_RPC_CONNECTION_PORT is not set}
+      INDEXER_PROCESSOR_POSTGRES_CONNECTION_STRING: postgres://postgres:password@postgres:5432/postgres
+      MOVEMENT_SYNC: ${MOVEMENT_SYNC:?MOVEMENT_SYNC is not set}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:?AWS_ACCESS_KEY_ID is not set}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:?AWS_SECRET_ACCESS_KEY is not set}
+      AWS_REGION: ${AWS_DEFAULT_REGION:?AWS_DEFAULT_REGION is not set}
+    volumes:
+      - ${DOT_MOVEMENT_PATH}:/.movement
+
+  celestia-light-node:
+    image: busybox
+    container_name: celestia-light-node
+    command: sh -c 'echo "Starting Celestia light-node services." && sleep infinity'
+    environment:
+      - DOT_MOVEMENT_PATH=/.movement
+    volumes:
+      - ${DOT_MOVEMENT_PATH}:/.movement
+    depends_on:
+      setup:
+        condition: service_healthy

networks/suzuka/setup/Cargo.toml

@@ -33,6 +33,8 @@ celestia-types = { workspace = true }
 async-recursion = { workspace = true }
 tracing-subscriber = { workspace = true }
 godfig = { workspace = true }
+syncup = { workspace = true }
+futures = { workspace = true }
 
 [dev-dependencies]
 tempfile = { workspace = true }

networks/suzuka/setup/src/main.rs

@@ -1,5 +1,7 @@
 use anyhow::Context;
 use godfig::{backend::config_file::ConfigFile, Godfig};
+use std::future::Future;
+use std::pin::Pin;
 use suzuka_config::Config;
 use suzuka_full_node_setup::{local::Local, SuzukaFullNodeSetupOperations};
 use tokio::signal::unix::signal;
@@ -39,6 +41,24 @@ async fn main() -> Result<(), anyhow::Error> {
 
 	// get the config file
 	let dot_movement = dot_movement::DotMovement::try_from_env()?;
+
+	// check if the MOVEMENT_SYNC environment variable is set
+	let sync_task: Pin<Box<dyn Future<Output = Result<(), anyhow::Error>> + Send>> =
+		if let Ok(bucket_arrow_glob) = std::env::var("MOVEMENT_SYNC") {
+			let mut bucket_arrow_glob = bucket_arrow_glob.split("<=>");
+			let bucket = bucket_arrow_glob.next().context(
+				"MOVEMENT_SYNC environment variable must be in the format <bucket>,<glob>",
+			)?;
+			let glob = bucket_arrow_glob.next().context(
+				"MOVEMENT_SYNC environment variable must be in the format <bucket>,<glob>",
+			)?;
+
+			let sync_task = dot_movement.sync(glob, bucket.to_string()).await?;
+			Box::pin(async { sync_task.await })
+		} else {
+			Box::pin(async { futures::future::pending::<Result<(), anyhow::Error>>().await })
+		};
+
 	let mut config_file = dot_movement.try_get_or_create_config_file().await?;
 
 	// get a matching godfig object
@@ -66,6 +86,10 @@ async fn main() -> Result<(), anyhow::Error> {
 		_ = stop_rx.changed() => {
 			tracing::info!("Cancellation received, killing anvil task.");
 		}
+		// sync task
+		_ = sync_task => {
+			tracing::info!("Sync task finished.");
+		}
 	}
 
 	Ok(())

protocol-units/da/m1/runners/src/celestia_bridge/local.rs

@@ -23,11 +23,13 @@ impl Local {
 		let max_attempts = 30;
 
 		// get the required connection details from the config
+		let connection_protocol = config.bridge.celestia_rpc_connection_protocol.clone();
 		let connection_hostname = config.bridge.celestia_rpc_connection_hostname.clone();
 		let connection_port = config.bridge.celestia_rpc_connection_port.clone();
-		let celestia_rpc_address = format!("{}:{}", connection_hostname, connection_port);
+		let celestia_rpc_address =
+			format!("{}://{}:{}", connection_protocol, connection_hostname, connection_port);
 
-		let first_block_request_url = format!("http://{}/block?height=1", celestia_rpc_address);
+		let first_block_request_url = format!("{}/block?height=1", celestia_rpc_address);
 		while genesis.len() <= 4 && cnt < max_attempts {
 			info!("Waiting for genesis block.");
 			let response = client
@@ -56,7 +58,7 @@ impl Local {
 
 	pub async fn run(
 		&self,
-		dot_movement: dot_movement::DotMovement,
+		_dot_movement: dot_movement::DotMovement,
 		config: m1_da_light_node_util::config::local::Config,
 	) -> Result<()> {
 		let genesis = self.get_genesis_block(&config).await?;

protocol-units/da/m1/runners/src/celestia_bridge/mod.rs

@@ -17,10 +17,10 @@ impl Runner for CelestiaBridge {
 				let local = local::Local::new();
 				local.run(dot_movement, config).await?;
 			}
-			m1_da_light_node_util::config::Config::Arabica(config) => {
+			m1_da_light_node_util::config::Config::Arabica(_config) => {
 				Err(anyhow::anyhow!("Arabica not implemented"))?;
 			}
-			m1_da_light_node_util::config::Config::Mocha(config) => {
+			m1_da_light_node_util::config::Config::Mocha(_config) => {
 				Err(anyhow::anyhow!("Mocha not implemented"))?;
 			}
 		}

protocol-units/da/m1/runners/src/celestia_light/arabica.rs

@@ -8,8 +8,8 @@ impl Arabica {
 
 	pub async fn run(
 		&self,
-		dot_movement: dot_movement::DotMovement,
-		config: m1_da_light_node_util::config::local::Config,
+		_dot_movement: dot_movement::DotMovement,
+		_config: m1_da_light_node_util::config::local::Config,
 	) -> Result<(), anyhow::Error> {
 		// celestia light start --core.ip validator-1.celestia-arabica-11.com --p2p.network arabica
 		commander::run_command(

protocol-units/da/m1/runners/src/celestia_light/mocha.rs

@@ -8,8 +8,8 @@ impl Mocha {
 
 	pub async fn run(
 		&self,
-		dot_movement: dot_movement::DotMovement,
-		config: m1_da_light_node_util::config::local::Config,
+		_dot_movement: dot_movement::DotMovement,
+		_config: m1_da_light_node_util::config::local::Config,
 	) -> Result<(), anyhow::Error> {
 		// celestia light start --core.ip validator-1.celestia-mocha-11.com --p2p.network mocha
 		commander::run_command(

protocol-units/da/m1/runners/src/celestia_light/mod.rs

@@ -14,7 +14,7 @@ impl Runner for CelestiaLight {
 		config: M1DaLightNodeConfig,
 	) -> Result<(), anyhow::Error> {
 		match config.m1_da_light_node_config {
-			m1_da_light_node_util::config::Config::Local(config) => {
+			m1_da_light_node_util::config::Config::Local(_config) => {
 				Err(anyhow::anyhow!("Local not implemented"))?;
 			}
 			m1_da_light_node_util::config::Config::Arabica(config) => {

protocol-units/da/m1/util/src/config/common.rs

@@ -12,6 +12,14 @@ env_default!(
 // The default port for the Celestia RPC
 env_default!(default_celestia_rpc_listen_port, "CELESTIA_RPC_LISTEN_PORT", u16, 26657);
 
+// The default Celestia RPC connection protocol
+env_default!(
+	default_celestia_rpc_connection_protocol,
+	"CELESTIA_RPC_CONNECTION_PROTOCOL",
+	String,
+	"http".to_string()
+);
+
 // The default Celestia RPC connection hostname
 env_default!(
 	default_celestia_rpc_connection_hostname,
@@ -34,6 +42,14 @@ env_default!(
 // The default port for the Celestia Node websocket
 env_default!(default_celestia_websocket_listen_port, "CELESTIA_WEBSOCKET_LISTEN_PORT", u16, 26658);
 
+// the default Celestia Node websocket connection protocol
+env_default!(
+	default_celestia_websocket_connection_protocol,
+	"CELESTIA_WEBSOCKET_CONNECTION_PROTOCOL",
+	String,
+	"ws".to_string()
+);
+
 // The default Celestia Node websocket connection hostname
 env_default!(
 	default_celestia_websocket_connection_hostname,

protocol-units/da/m1/util/src/config/local/appd.rs

@@ -2,7 +2,7 @@ use crate::config::common::{
 	default_celestia_appd_replace_args, default_celestia_appd_use_replace_args,
 	default_celestia_chain_id, default_celestia_namespace, default_celestia_rpc_listen_hostname,
 	default_celestia_rpc_listen_port, default_celestia_websocket_connection_hostname,
-	default_celestia_websocket_connection_port,
+	default_celestia_websocket_connection_port, default_celestia_websocket_connection_protocol,
 };
 
 use celestia_types::nmt::Namespace;
@@ -19,6 +19,10 @@ pub struct Config {
 	#[serde(default = "default_celestia_rpc_listen_port")]
 	pub celestia_rpc_listen_port: u16,
 
+	/// The protocol for the Celestia Node websocket
+	#[serde(default = "default_celestia_websocket_connection_protocol")]
+	pub celestia_websocket_connection_protocol: String,
+
 	/// The hostname of the Celestia Node websocket
 	#[serde(default = "default_celestia_websocket_connection_hostname")]
 	pub celestia_websocket_connection_hostname: String,
@@ -60,6 +64,8 @@ impl Default for Config {
 		Self {
 			celestia_rpc_listen_hostname: default_celestia_rpc_listen_hostname(),
 			celestia_rpc_listen_port: default_celestia_rpc_listen_port(),
+			celestia_websocket_connection_protocol: default_celestia_websocket_connection_protocol(
+			),
 			celestia_websocket_connection_hostname: default_celestia_websocket_connection_hostname(
 			),
 			celestia_websocket_connection_port: default_celestia_websocket_connection_port(),