From adab48f770b84667aaff512efb72a9c36d4f036f Mon Sep 17 00:00:00 2001 From: int88 Date: Thu, 13 Apr 2023 17:19:25 +0800 Subject: [PATCH 1/4] use state cache to avoid unnecessary block replay --- beacon_node/src/cli.rs | 7 ++++ beacon_node/src/config.rs | 6 +++ beacon_node/store/src/config.rs | 4 ++ beacon_node/store/src/hot_cold_store.rs | 54 ++++++++++++++++++++----- book/src/api-lighthouse.md | 1 + book/src/database-migrations.md | 1 + lighthouse/tests/beacon_node.rs | 7 ++++ 7 files changed, 71 insertions(+), 9 deletions(-) diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 72a5dda9522..dcf964a360e 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -511,6 +511,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .help("Specifies how many blocks the database should cache in memory [default: 5]") .takes_value(true) ) + .arg( + Arg::with_name("state-cache-size") + .long("state-cache-size") + .value_name("SIZE") + .help("Specifies how many states the database should cache in memory [default: 1]") + .takes_value(true) + ) /* * Execution Layer Integration */ diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 6ad1fea3b27..586213b6873 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -380,6 +380,12 @@ pub fn get_config( .map_err(|_| "block-cache-size is not a valid integer".to_string())?; } + if let Some(state_cache_size) = cli_args.value_of("state-cache-size") { + client_config.store.state_cache_size = state_cache_size + .parse() + .map_err(|_| "state-cache-size is not a valid integer".to_string())?; + } + client_config.store.compact_on_init = cli_args.is_present("compact-db"); if let Some(compact_on_prune) = cli_args.value_of("auto-compact-db") { client_config.store.compact_on_prune = compact_on_prune diff --git a/beacon_node/store/src/config.rs b/beacon_node/store/src/config.rs index 027b8152ee5..0f1ce96b412 100644 --- a/beacon_node/store/src/config.rs +++ b/beacon_node/store/src/config.rs @@ -7,6 +7,7 @@ use types::{EthSpec, MinimalEthSpec}; pub const PREV_DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 2048; pub const DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 8192; pub const DEFAULT_BLOCK_CACHE_SIZE: usize = 5; +pub const DEFAULT_STATE_CACHE_SIZE: usize = 1; /// Database configuration parameters. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -17,6 +18,8 @@ pub struct StoreConfig { pub slots_per_restore_point_set_explicitly: bool, /// Maximum number of blocks to store in the in-memory block cache. pub block_cache_size: usize, + /// Maximum number of states to store in the in-memory state cache. + pub state_cache_size: usize, /// Whether to compact the database on initialization. pub compact_on_init: bool, /// Whether to compact the database during database pruning. @@ -43,6 +46,7 @@ impl Default for StoreConfig { slots_per_restore_point: MinimalEthSpec::slots_per_historical_root() as u64, slots_per_restore_point_set_explicitly: false, block_cache_size: DEFAULT_BLOCK_CACHE_SIZE, + state_cache_size: DEFAULT_STATE_CACHE_SIZE, compact_on_init: false, compact_on_prune: true, prune_payloads: true, diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 02608f9a0bd..f3033533467 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -40,7 +40,7 @@ use std::sync::Arc; use std::time::Duration; use types::*; -/// On-disk database that stores finalized states efficiently. +/// On-disk database that stores fnalized states efficiently. /// /// Stores vector fields like the `block_roots` and `state_roots` separately, and only stores /// intermittent "restore point" states pre-finalization. @@ -62,6 +62,8 @@ pub struct HotColdDB, Cold: ItemStore> { pub hot_db: Hot, /// LRU cache of deserialized blocks. Updated whenever a block is loaded. block_cache: Mutex>>, + /// LRU cache of replayed states. + state_cache: Mutex>>, /// Chain spec. pub(crate) spec: ChainSpec, /// Logger. @@ -129,6 +131,7 @@ impl HotColdDB, MemoryStore> { cold_db: MemoryStore::open(), hot_db: MemoryStore::open(), block_cache: Mutex::new(LruCache::new(config.block_cache_size)), + state_cache: Mutex::new(LruCache::new(config.state_cache_size)), config, spec, log, @@ -162,6 +165,7 @@ impl HotColdDB, LevelDB> { cold_db: LevelDB::open(cold_path)?, hot_db: LevelDB::open(hot_path)?, block_cache: Mutex::new(LruCache::new(config.block_cache_size)), + state_cache: Mutex::new(LruCache::new(config.state_cache_size)), config, spec, log, @@ -579,6 +583,9 @@ impl, Cold: ItemStore> HotColdDB /// (which are frozen, and won't be deleted), or valid descendents of the finalized checkpoint /// (which will be deleted by this function but shouldn't be). pub fn delete_state(&self, state_root: &Hash256, slot: Slot) -> Result<(), Error> { + // Delete the state from the cache. + self.state_cache.lock().pop(&slot); + // Delete the state summary. self.hot_db .key_delete(DBColumn::BeaconStateSummary.into(), state_root.as_bytes())?; @@ -977,40 +984,69 @@ impl, Cold: ItemStore> HotColdDB /// Load a frozen state that lies between restore points. fn load_cold_intermediate_state(&self, slot: Slot) -> Result, Error> { + if let Some(state) = self.state_cache.lock().get(&slot) { + return Ok(state.clone()); + } + // 1. Load the restore points either side of the intermediate state. let low_restore_point_idx = slot.as_u64() / self.config.slots_per_restore_point; let high_restore_point_idx = low_restore_point_idx + 1; + // Use low restore point as the base state. + let mut low_slot: Slot = + Slot::new(low_restore_point_idx * self.config.slots_per_restore_point); + let mut low_state: Option> = None; + + // Try to get a more recent state from the cache to avoid massive blocks replay. + for (s, state) in self.state_cache.lock().iter() { + if s.as_u64() / self.config.slots_per_restore_point == low_restore_point_idx + && *s < slot + && low_slot < *s + { + low_slot = *s; + low_state = Some(state.clone()); + } + } + + // If low_state is still None, use load_restore_point_by_index to load the state. + if low_state.is_none() { + low_state = Some(self.load_restore_point_by_index(low_restore_point_idx)?); + } + // Acquire the read lock, so that the split can't change while this is happening. let split = self.split.read_recursive(); - let low_restore_point = self.load_restore_point_by_index(low_restore_point_idx)?; let high_restore_point = self.get_restore_point(high_restore_point_idx, &split)?; - // 2. Load the blocks from the high restore point back to the low restore point. + // 2. Load the blocks from the high restore point back to the low point. let blocks = self.load_blocks_to_replay( - low_restore_point.slot(), + low_slot, slot, self.get_high_restore_point_block_root(&high_restore_point, slot)?, )?; - // 3. Replay the blocks on top of the low restore point. + // 3. Replay the blocks on top of the low point. // Use a forwards state root iterator to avoid doing any tree hashing. // The state root of the high restore point should never be used, so is safely set to 0. let state_root_iter = self.forwards_state_roots_iterator_until( - low_restore_point.slot(), + low_slot, slot, || (high_restore_point, Hash256::zero()), &self.spec, )?; - self.replay_blocks( - low_restore_point, + let state = self.replay_blocks( + low_state.unwrap(), blocks, slot, Some(state_root_iter), StateRootStrategy::Accurate, - ) + )?; + + // If state is not error, put it in the cache. + self.state_cache.lock().put(slot, state.clone()); + + Ok(state) } /// Get the restore point with the given index, or if it is out of bounds, the split state. diff --git a/book/src/api-lighthouse.md b/book/src/api-lighthouse.md index 28481809703..cc6a2381f41 100644 --- a/book/src/api-lighthouse.md +++ b/book/src/api-lighthouse.md @@ -455,6 +455,7 @@ curl "http://localhost:5052/lighthouse/database/info" | jq "config": { "slots_per_restore_point": 2048, "block_cache_size": 5, + "state_cache_size": 1, "compact_on_init": false, "compact_on_prune": true }, diff --git a/book/src/database-migrations.md b/book/src/database-migrations.md index d2b7b518d75..76665ec72d2 100644 --- a/book/src/database-migrations.md +++ b/book/src/database-migrations.md @@ -92,6 +92,7 @@ curl "http://localhost:5052/lighthouse/database/info" "slots_per_restore_point": 8192, "slots_per_restore_point_set_explicitly": true, "block_cache_size": 5, + "state_cache_size": 1, "compact_on_init": false, "compact_on_prune": true } diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 64646a6c573..e5a79781233 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1634,6 +1634,13 @@ fn block_cache_size_flag() { .with_config(|config| assert_eq!(config.store.block_cache_size, 4_usize)); } #[test] +fn state_cache_size_flag() { + CommandLineTest::new() + .flag("state-cache-size", Some("4")) + .run_with_zero_port() + .with_config(|config| assert_eq!(config.store.state_cache_size, 4_usize)); +} +#[test] fn auto_compact_db_flag() { CommandLineTest::new() .flag("auto-compact-db", Some("false")) From 845f9a1478bd285b7f62e525ae94b7f094cbf470 Mon Sep 17 00:00:00 2001 From: int88 Date: Fri, 28 Apr 2023 10:18:07 +0800 Subject: [PATCH 2/4] rename state-cache-size to historic-state-cache-size --- beacon_node/src/cli.rs | 6 +++--- beacon_node/src/config.rs | 6 +++--- beacon_node/store/src/config.rs | 8 ++++---- beacon_node/store/src/hot_cold_store.rs | 4 ++-- book/src/api-lighthouse.md | 2 +- book/src/database-migrations.md | 2 +- lighthouse/tests/beacon_node.rs | 6 +++--- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index dcf964a360e..48a401dfc6a 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -512,10 +512,10 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .takes_value(true) ) .arg( - Arg::with_name("state-cache-size") - .long("state-cache-size") + Arg::with_name("historic-state-cache-size") + .long("historic-state-cache-size") .value_name("SIZE") - .help("Specifies how many states the database should cache in memory [default: 1]") + .help("Specifies how many states from the freezer database should cache in memory [default: 1]") .takes_value(true) ) /* diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 586213b6873..aec75fd4a4a 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -380,10 +380,10 @@ pub fn get_config( .map_err(|_| "block-cache-size is not a valid integer".to_string())?; } - if let Some(state_cache_size) = cli_args.value_of("state-cache-size") { - client_config.store.state_cache_size = state_cache_size + if let Some(historic_state_cache_size) = cli_args.value_of("historic-state-cache-size") { + client_config.store.historic_state_cache_size = historic_state_cache_size .parse() - .map_err(|_| "state-cache-size is not a valid integer".to_string())?; + .map_err(|_| "historic-state-cache-size is not a valid integer".to_string())?; } client_config.store.compact_on_init = cli_args.is_present("compact-db"); diff --git a/beacon_node/store/src/config.rs b/beacon_node/store/src/config.rs index 0f1ce96b412..581003b4fae 100644 --- a/beacon_node/store/src/config.rs +++ b/beacon_node/store/src/config.rs @@ -7,7 +7,7 @@ use types::{EthSpec, MinimalEthSpec}; pub const PREV_DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 2048; pub const DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 8192; pub const DEFAULT_BLOCK_CACHE_SIZE: usize = 5; -pub const DEFAULT_STATE_CACHE_SIZE: usize = 1; +pub const DEFAULT_HISTORIC_STATE_CACHE_SIZE: usize = 1; /// Database configuration parameters. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -18,8 +18,8 @@ pub struct StoreConfig { pub slots_per_restore_point_set_explicitly: bool, /// Maximum number of blocks to store in the in-memory block cache. pub block_cache_size: usize, - /// Maximum number of states to store in the in-memory state cache. - pub state_cache_size: usize, + /// Maximum number of states from freezer database to store in the in-memory state cache. + pub historic_state_cache_size: usize, /// Whether to compact the database on initialization. pub compact_on_init: bool, /// Whether to compact the database during database pruning. @@ -46,7 +46,7 @@ impl Default for StoreConfig { slots_per_restore_point: MinimalEthSpec::slots_per_historical_root() as u64, slots_per_restore_point_set_explicitly: false, block_cache_size: DEFAULT_BLOCK_CACHE_SIZE, - state_cache_size: DEFAULT_STATE_CACHE_SIZE, + historic_state_cache_size: DEFAULT_HISTORIC_STATE_CACHE_SIZE, compact_on_init: false, compact_on_prune: true, prune_payloads: true, diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index f3033533467..ae12ada2c5b 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -131,7 +131,7 @@ impl HotColdDB, MemoryStore> { cold_db: MemoryStore::open(), hot_db: MemoryStore::open(), block_cache: Mutex::new(LruCache::new(config.block_cache_size)), - state_cache: Mutex::new(LruCache::new(config.state_cache_size)), + state_cache: Mutex::new(LruCache::new(config.historic_state_cache_size)), config, spec, log, @@ -165,7 +165,7 @@ impl HotColdDB, LevelDB> { cold_db: LevelDB::open(cold_path)?, hot_db: LevelDB::open(hot_path)?, block_cache: Mutex::new(LruCache::new(config.block_cache_size)), - state_cache: Mutex::new(LruCache::new(config.state_cache_size)), + state_cache: Mutex::new(LruCache::new(config.historic_state_cache_size)), config, spec, log, diff --git a/book/src/api-lighthouse.md b/book/src/api-lighthouse.md index cc6a2381f41..aa4a618b145 100644 --- a/book/src/api-lighthouse.md +++ b/book/src/api-lighthouse.md @@ -455,7 +455,7 @@ curl "http://localhost:5052/lighthouse/database/info" | jq "config": { "slots_per_restore_point": 2048, "block_cache_size": 5, - "state_cache_size": 1, + "historic_state_cache_size": 1, "compact_on_init": false, "compact_on_prune": true }, diff --git a/book/src/database-migrations.md b/book/src/database-migrations.md index 76665ec72d2..5e0b8963590 100644 --- a/book/src/database-migrations.md +++ b/book/src/database-migrations.md @@ -92,7 +92,7 @@ curl "http://localhost:5052/lighthouse/database/info" "slots_per_restore_point": 8192, "slots_per_restore_point_set_explicitly": true, "block_cache_size": 5, - "state_cache_size": 1, + "historic_state_cache_size": 1, "compact_on_init": false, "compact_on_prune": true } diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index e5a79781233..74cb9ef1d13 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1634,11 +1634,11 @@ fn block_cache_size_flag() { .with_config(|config| assert_eq!(config.store.block_cache_size, 4_usize)); } #[test] -fn state_cache_size_flag() { +fn historic_state_cache_size_flag() { CommandLineTest::new() - .flag("state-cache-size", Some("4")) + .flag("historic-state-cache-size", Some("4")) .run_with_zero_port() - .with_config(|config| assert_eq!(config.store.state_cache_size, 4_usize)); + .with_config(|config| assert_eq!(config.store.historic_state_cache_size, 4_usize)); } #[test] fn auto_compact_db_flag() { From 117439fc1a772b68efb50c857e5b36f8b78c00c5 Mon Sep 17 00:00:00 2001 From: int88 Date: Thu, 4 May 2023 15:13:34 +0800 Subject: [PATCH 3/4] some optimization --- beacon_node/store/src/hot_cold_store.rs | 14 ++++++-------- book/src/advanced_database.md | 11 +++++++++++ lighthouse/tests/beacon_node.rs | 12 ++++++++++++ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index ae12ada2c5b..70fb22742e0 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -40,7 +40,7 @@ use std::sync::Arc; use std::time::Duration; use types::*; -/// On-disk database that stores fnalized states efficiently. +/// On-disk database that stores finalized states efficiently. /// /// Stores vector fields like the `block_roots` and `state_roots` separately, and only stores /// intermittent "restore point" states pre-finalization. @@ -583,9 +583,6 @@ impl, Cold: ItemStore> HotColdDB /// (which are frozen, and won't be deleted), or valid descendents of the finalized checkpoint /// (which will be deleted by this function but shouldn't be). pub fn delete_state(&self, state_root: &Hash256, slot: Slot) -> Result<(), Error> { - // Delete the state from the cache. - self.state_cache.lock().pop(&slot); - // Delete the state summary. self.hot_db .key_delete(DBColumn::BeaconStateSummary.into(), state_root.as_bytes())?; @@ -1009,9 +1006,10 @@ impl, Cold: ItemStore> HotColdDB } // If low_state is still None, use load_restore_point_by_index to load the state. - if low_state.is_none() { - low_state = Some(self.load_restore_point_by_index(low_restore_point_idx)?); - } + let low_state = match low_state { + Some(state) => state, + None => self.load_restore_point_by_index(low_restore_point_idx)?, + }; // Acquire the read lock, so that the split can't change while this is happening. let split = self.split.read_recursive(); @@ -1036,7 +1034,7 @@ impl, Cold: ItemStore> HotColdDB )?; let state = self.replay_blocks( - low_state.unwrap(), + low_state, blocks, slot, Some(state_root_iter), diff --git a/book/src/advanced_database.md b/book/src/advanced_database.md index 397d9a28b5f..ae8acbdd13e 100644 --- a/book/src/advanced_database.md +++ b/book/src/advanced_database.md @@ -58,6 +58,17 @@ the `--slots-per-restore-point` flag: lighthouse beacon_node --slots-per-restore-point 32 ``` +### Historic state cache + +To avoid repeatedly reconstruct the states from the restore points when searching for sequential historical states. +Lighthouse temporarily cache the reconstructed state which could prevent significant state reconstruction to optimize +this kind of requests. The historical state cache size could be specified with flag `--historic-state-cache-size` (default +value is 1): + +```bash +lighthouse beacon_node --historic-state-cache-size 4 +``` + ## Glossary * _Freezer DB_: part of the database storing finalized states. States are stored in a sparser diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 74cb9ef1d13..ffac8ade7e1 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1641,6 +1641,18 @@ fn historic_state_cache_size_flag() { .with_config(|config| assert_eq!(config.store.historic_state_cache_size, 4_usize)); } #[test] +fn historic_state_cache_size_default() { + use beacon_node::beacon_chain::store::config::DEFAULT_HISTORIC_STATE_CACHE_SIZE; + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.store.historic_state_cache_size, + DEFAULT_HISTORIC_STATE_CACHE_SIZE + ); + }); +} +#[test] fn auto_compact_db_flag() { CommandLineTest::new() .flag("auto-compact-db", Some("false")) From 32be2eee9d3cd775f6bb11e2944c7adbd351c97b Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 5 May 2023 10:49:12 +1000 Subject: [PATCH 4/4] Tweak docs --- book/src/advanced_database.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/book/src/advanced_database.md b/book/src/advanced_database.md index ae8acbdd13e..57e49531ca0 100644 --- a/book/src/advanced_database.md +++ b/book/src/advanced_database.md @@ -60,10 +60,9 @@ lighthouse beacon_node --slots-per-restore-point 32 ### Historic state cache -To avoid repeatedly reconstruct the states from the restore points when searching for sequential historical states. -Lighthouse temporarily cache the reconstructed state which could prevent significant state reconstruction to optimize -this kind of requests. The historical state cache size could be specified with flag `--historic-state-cache-size` (default -value is 1): +Lighthouse includes a cache to avoid repeatedly replaying blocks when loading historic states. Lighthouse will cache a limited number of reconstructed states and will re-use them when serving requests for subsequent states at higher slots. This greatly reduces the cost of requesting several states in order, and we recommend that applications like block explorers take advantage of this cache. + +The historical state cache size can be specified with the flag `--historic-state-cache-size` (default value is 1): ```bash lighthouse beacon_node --historic-state-cache-size 4