feat: get l2 block number from storage write logs (#110)

* feat: get l2 block number from storage write logs

* doc: make comment into code doc

* doc: clarify `l1_batch_number`

* chore: don't panic on missing l2 batch number

* fix: set l1 batch number not l2 batch number

Author: zeapoz

src/processor/snapshot/exporter.rs

@@ -1,8 +1,7 @@
 use std::path::{Path, PathBuf};
 
-use ethers::types::U256;
-use eyre::Result;
-use state_reconstruct_fetcher::constants::ethereum::GENESIS_BLOCK;
+use ethers::types::{U256, U64};
+use eyre::{OptionExt, Result};
 use state_reconstruct_storage::{
     snapshot::SnapshotDatabase,
     snapshot_columns,
@@ -37,14 +36,21 @@ impl SnapshotExporter {
     }
 
     pub fn export_snapshot(&self, num_chunks: usize) -> Result<()> {
-        let latest_l1_batch_number = self.database.get_latest_l1_batch_number()?;
-        // L1 batch number is calculated from the batch number where the
-        // DiamondProxy contract was deployed (`GENESIS_BLOCK`).
-        let l1_batch_number = latest_l1_batch_number - GENESIS_BLOCK;
-        let l2_batch_number = self.database.get_latest_l2_batch_number()?;
+        let l2_batch_number = self
+            .database
+            .get_latest_l2_batch_number()?
+            .ok_or_eyre("no latest l2 batch number in snapshot db")?;
+        let l2_block_number = self.database.get_latest_l2_block_number()?.unwrap_or({
+            tracing::warn!("WARNING: the database contains no l2 block number entry and will not be compatible with the ZKSync External Node! To export a compatible snapshot, please let the prepare-snapshot command run until an l2 block number can be found.");
+            U64::from(0)
+        });
         let mut header = SnapshotHeader {
-            l1_batch_number: l1_batch_number.as_u64(),
-            miniblock_number: l2_batch_number.as_u64(),
+            // NOTE: `l1_batch_number` in the snapshot header actually refers
+            // to the ZKsync batch number and not the Ethereum batch height we
+            // store in the snapshot database. In the snapshot database this
+            // field is referred to as `l2_batch_number`.
+            l1_batch_number: l2_batch_number.as_u64(),
+            miniblock_number: l2_block_number.as_u64(),
             ..Default::default()
         };
 

src/processor/snapshot/mod.rs

@@ -10,7 +10,7 @@ use blake2::{Blake2s256, Digest};
 use ethers::types::{Address, H256, U256, U64};
 use eyre::Result;
 use state_reconstruct_fetcher::{
-    constants::{ethereum, storage},
+    constants::{ethereum, storage, zksync::L2_BLOCK_NUMBER_ADDRESS},
     types::CommitBlock,
 };
 use state_reconstruct_storage::{
@@ -20,6 +20,7 @@ use state_reconstruct_storage::{
 use tokio::sync::mpsc;
 
 use super::Processor;
+use crate::util::{h256_to_u256, unpack_block_info};
 
 pub const DEFAULT_DB_PATH: &str = "snapshot_db";
 pub const SNAPSHOT_HEADER_FILE_NAME: &str = "snapshot-header.json";
@@ -54,7 +55,9 @@ impl SnapshotBuilder {
 
     // Gets the next L1 batch number to be processed for ues in state recovery.
     pub fn get_latest_l1_batch_number(&self) -> Result<U64> {
-        self.database.get_latest_l1_batch_number()
+        self.database
+            .get_latest_l1_batch_number()
+            .map(|o| o.unwrap_or(U64::from(0)))
     }
 }
 
@@ -93,6 +96,14 @@ impl Processor for SnapshotBuilder {
                     .process_value(U256::from_big_endian(&key[0..32]), *value)
                     .expect("failed to get key from database");
 
+                // We make sure to track writes to the L2 block number address.
+                if hex::encode(key) == L2_BLOCK_NUMBER_ADDRESS {
+                    let (block_number, _timestamp) = unpack_block_info(h256_to_u256(value));
+                    self.database
+                        .set_latest_l2_block_number(block_number)
+                        .expect("failed to insert latest l2 block number");
+                }
+
                 if self
                     .database
                     .update_storage_log_value(index as u64, &value.to_fixed_bytes())
@@ -122,7 +133,7 @@ impl Processor for SnapshotBuilder {
 
             let _ = self
                 .database
-                .set_latest_l2_batch_number(block.l2_block_number);
+                .set_latest_l2_block_number(block.l2_block_number);
 
             if let Some(number) = block.l1_block_number {
                 let _ = self.database.set_latest_l1_batch_number(number);

src/util/mod.rs

@@ -1 +1,16 @@
+use primitive_types::{H256, U256};
+
 pub mod json;
+
+pub const SYSTEM_BLOCK_INFO_BLOCK_NUMBER_MULTIPLIER: U256 = U256([0, 0, 1, 0]);
+
+pub fn h256_to_u256(num: H256) -> U256 {
+    U256::from_big_endian(num.as_bytes())
+}
+
+/// Returns block.number/timestamp based on the block's information
+pub fn unpack_block_info(info: U256) -> (u64, u64) {
+    let block_number = (info / SYSTEM_BLOCK_INFO_BLOCK_NUMBER_MULTIPLIER).as_u64();
+    let block_timestamp = (info % SYSTEM_BLOCK_INFO_BLOCK_NUMBER_MULTIPLIER).as_u64();
+    (block_number, block_timestamp)
+}

state-reconstruct-fetcher/src/constants.rs

@@ -32,12 +32,16 @@ pub mod storage {
 pub mod zksync {
     /// Bytes in raw L2 to L1 log.
     pub const L2_TO_L1_LOG_SERIALIZE_SIZE: usize = 88;
-    // The bitmask by applying which to the compressed state diff metadata we retrieve its operation.
+    /// The bitmask by applying which to the compressed state diff metadata we retrieve its operation.
     pub const OPERATION_BITMASK: u8 = 7;
-    // The number of bits shifting the compressed state diff metadata by which we retrieve its length.
+    /// The number of bits shifting the compressed state diff metadata by which we retrieve its length.
     pub const LENGTH_BITS_OFFSET: u8 = 3;
-    // Size of `CommitBatchInfo.pubdataCommitments` item.
+    /// Size of `CommitBatchInfo.pubdataCommitments` item.
     pub const PUBDATA_COMMITMENT_SIZE: usize = 144;
-    // The number of trailing bytes to ignore when using calldata post-blobs. Contains unused blob commitments.
+    /// The number of trailing bytes to ignore when using calldata post-blobs. Contains unused blob commitments.
     pub const CALLDATA_SOURCE_TAIL_SIZE: usize = 32;
+
+    /// The storage address where the latest L2 block number is written to.
+    pub const L2_BLOCK_NUMBER_ADDRESS: &str =
+        "5e5a67d1b864c576f39bb2b77c6537744c0f03515abce63b473bb7c56ad07d8e";
 }

state-reconstruct-storage/src/lib.rs

@@ -24,10 +24,12 @@ pub mod snapshot_columns {
     pub const FACTORY_DEPS: &str = "factory_deps";
 
     pub const LAST_REPEATED_KEY_INDEX: &str = "SNAPSHOT_LAST_REPEATED_KEY_INDEX";
-    /// The latest l1 block number that was processed.
+    /// The latest l1 batch number that was processed.
     pub const LATEST_L1_BATCH: &str = "SNAPSHOT_LATEST_L1_BATCH";
-    /// The latest l2 block number that was processed.
+    /// The latest l2 batch number that was processed.
     pub const LATEST_L2_BATCH: &str = "SNAPSHOT_LATEST_L2_BATCH";
+    /// The latest l2 block number that was processed.
+    pub const LATEST_L2_BLOCK: &str = "SNAPSHOT_LATEST_L2_BLOCK";
 }
 
 // NOTE: This is moved here as a temporary measure to resolve a cyclic dependency issue.

state-reconstruct-storage/src/snapshot.rs

@@ -38,6 +38,7 @@ impl SnapshotDatabase {
                 snapshot_columns::FACTORY_DEPS,
                 snapshot_columns::LATEST_L1_BATCH,
                 snapshot_columns::LATEST_L2_BATCH,
+                snapshot_columns::LATEST_L2_BLOCK,
             ],
         )?;
 
@@ -60,6 +61,7 @@ impl SnapshotDatabase {
                 snapshot_columns::FACTORY_DEPS,
                 snapshot_columns::LATEST_L1_BATCH,
                 snapshot_columns::LATEST_L2_BATCH,
+                snapshot_columns::LATEST_L2_BLOCK,
             ],
             false,
         )?;
@@ -154,50 +156,56 @@ impl SnapshotDatabase {
             .map_err(Into::into)
     }
 
-    pub fn get_latest_l1_batch_number(&self) -> Result<U64> {
+    pub fn get_latest_l1_batch_number(&self) -> Result<Option<U64>> {
         self.get_metadata_value(snapshot_columns::LATEST_L1_BATCH)
-            .map(U64::from)
+            .map(|o| o.map(U64::from))
     }
 
     pub fn set_latest_l1_batch_number(&self, number: u64) -> Result<()> {
         self.set_metadata_value(snapshot_columns::LATEST_L1_BATCH, number)
     }
 
-    pub fn get_latest_l2_batch_number(&self) -> Result<U64> {
+    pub fn get_latest_l2_batch_number(&self) -> Result<Option<U64>> {
         self.get_metadata_value(snapshot_columns::LATEST_L2_BATCH)
-            .map(U64::from)
+            .map(|o| o.map(U64::from))
     }
 
     pub fn set_latest_l2_batch_number(&self, number: u64) -> Result<()> {
         self.set_metadata_value(snapshot_columns::LATEST_L2_BATCH, number)
     }
 
+    pub fn get_latest_l2_block_number(&self) -> Result<Option<U64>> {
+        self.get_metadata_value(snapshot_columns::LATEST_L2_BLOCK)
+            .map(|o| o.map(U64::from))
+    }
+
+    pub fn set_latest_l2_block_number(&self, number: u64) -> Result<()> {
+        self.set_metadata_value(snapshot_columns::LATEST_L2_BLOCK, number)
+    }
+
     pub fn get_last_repeated_key_index(&self) -> Result<u64> {
         self.get_metadata_value(snapshot_columns::LAST_REPEATED_KEY_INDEX)
+            .map(|o| o.unwrap_or(0))
     }
 
     pub fn set_last_repeated_key_index(&self, idx: u64) -> Result<()> {
         self.set_metadata_value(snapshot_columns::LAST_REPEATED_KEY_INDEX, idx)
     }
 
-    fn get_metadata_value(&self, value_name: &str) -> Result<u64> {
+    fn get_metadata_value(&self, value_name: &str) -> Result<Option<u64>> {
         let metadata = self.cf_handle(METADATA).unwrap();
-        Ok(
-            if let Some(idx_bytes) = self.get_cf(metadata, value_name)? {
-                u64::from_be_bytes([
-                    idx_bytes[0],
-                    idx_bytes[1],
-                    idx_bytes[2],
-                    idx_bytes[3],
-                    idx_bytes[4],
-                    idx_bytes[5],
-                    idx_bytes[6],
-                    idx_bytes[7],
-                ])
-            } else {
-                0
-            },
-        )
+        Ok(self.get_cf(metadata, value_name)?.map(|idx_bytes| {
+            u64::from_be_bytes([
+                idx_bytes[0],
+                idx_bytes[1],
+                idx_bytes[2],
+                idx_bytes[3],
+                idx_bytes[4],
+                idx_bytes[5],
+                idx_bytes[6],
+                idx_bytes[7],
+            ])
+        }))
     }
 
     fn set_metadata_value(&self, value_name: &str, value: u64) -> Result<()> {