Merge branch 'kaspanet:master' into master

Author: supr

Cargo.lock

@@ -63,7 +63,7 @@ members = [
 
 [workspace.package]
 rust-version = "1.82.0"
-version = "0.17.0"
+version = "0.17.1"
 authors = ["Kaspa developers"]
 license = "ISC"
 repository = "https://github.com/kaspanet/rusty-kaspa"
@@ -80,61 +80,61 @@ include = [
 ]
 
 [workspace.dependencies]
-# kaspa-testing-integration = { version = "0.17.0", path = "testing/integration" }
-kaspa-addresses = { version = "0.17.0", path = "crypto/addresses" }
-kaspa-addressmanager = { version = "0.17.0", path = "components/addressmanager" }
-kaspa-bip32 = { version = "0.17.0", path = "wallet/bip32" }
-kaspa-cli = { version = "0.17.0", path = "cli" }
-kaspa-connectionmanager = { version = "0.17.0", path = "components/connectionmanager" }
-kaspa-consensus = { version = "0.17.0", path = "consensus" }
-kaspa-consensus-core = { version = "0.17.0", path = "consensus/core" }
-kaspa-consensus-client = { version = "0.17.0", path = "consensus/client" }
-kaspa-consensus-notify = { version = "0.17.0", path = "consensus/notify" }
-kaspa-consensus-wasm = { version = "0.17.0", path = "consensus/wasm" }
-kaspa-consensusmanager = { version = "0.17.0", path = "components/consensusmanager" }
-kaspa-core = { version = "0.17.0", path = "core" }
-kaspa-daemon = { version = "0.17.0", path = "daemon" }
-kaspa-database = { version = "0.17.0", path = "database" }
-kaspa-grpc-client = { version = "0.17.0", path = "rpc/grpc/client" }
-kaspa-grpc-core = { version = "0.17.0", path = "rpc/grpc/core" }
-kaspa-grpc-server = { version = "0.17.0", path = "rpc/grpc/server" }
-kaspa-hashes = { version = "0.17.0", path = "crypto/hashes" }
-kaspa-index-core = { version = "0.17.0", path = "indexes/core" }
-kaspa-index-processor = { version = "0.17.0", path = "indexes/processor" }
-kaspa-math = { version = "0.17.0", path = "math" }
-kaspa-merkle = { version = "0.17.0", path = "crypto/merkle" }
-kaspa-metrics-core = { version = "0.17.0", path = "metrics/core" }
-kaspa-mining = { version = "0.17.0", path = "mining" }
-kaspa-mining-errors = { version = "0.17.0", path = "mining/errors" }
-kaspa-muhash = { version = "0.17.0", path = "crypto/muhash" }
-kaspa-notify = { version = "0.17.0", path = "notify" }
-kaspa-p2p-flows = { version = "0.17.0", path = "protocol/flows" }
-kaspa-p2p-lib = { version = "0.17.0", path = "protocol/p2p" }
-kaspa-perf-monitor = { version = "0.17.0", path = "metrics/perf_monitor" }
-kaspa-pow = { version = "0.17.0", path = "consensus/pow" }
-kaspa-rpc-core = { version = "0.17.0", path = "rpc/core" }
-kaspa-rpc-macros = { version = "0.17.0", path = "rpc/macros" }
-kaspa-rpc-service = { version = "0.17.0", path = "rpc/service" }
-kaspa-txscript = { version = "0.17.0", path = "crypto/txscript" }
-kaspa-txscript-errors = { version = "0.17.0", path = "crypto/txscript/errors" }
-kaspa-utils = { version = "0.17.0", path = "utils" }
-kaspa-utils-tower = { version = "0.17.0", path = "utils/tower" }
-kaspa-utxoindex = { version = "0.17.0", path = "indexes/utxoindex" }
-kaspa-wallet = { version = "0.17.0", path = "wallet/native" }
-kaspa-wallet-cli-wasm = { version = "0.17.0", path = "wallet/wasm" }
-kaspa-wallet-keys = { version = "0.17.0", path = "wallet/keys" }
-kaspa-wallet-pskt = { version = "0.17.0", path = "wallet/pskt" }
-kaspa-wallet-core = { version = "0.17.0", path = "wallet/core" }
-kaspa-wallet-macros = { version = "0.17.0", path = "wallet/macros" }
-kaspa-wasm = { version = "0.17.0", path = "wasm" }
-kaspa-wasm-core = { version = "0.17.0", path = "wasm/core" }
-kaspa-wrpc-client = { version = "0.17.0", path = "rpc/wrpc/client" }
-kaspa-wrpc-proxy = { version = "0.17.0", path = "rpc/wrpc/proxy" }
-kaspa-wrpc-server = { version = "0.17.0", path = "rpc/wrpc/server" }
-kaspa-wrpc-wasm = { version = "0.17.0", path = "rpc/wrpc/wasm" }
-kaspa-wrpc-example-subscriber = { version = "0.17.0", path = "rpc/wrpc/examples/subscriber" }
-kaspad = { version = "0.17.0", path = "kaspad" }
-kaspa-alloc = { version = "0.17.0", path = "utils/alloc" }
+# kaspa-testing-integration = { version = "0.17.1", path = "testing/integration" }
+kaspa-addresses = { version = "0.17.1", path = "crypto/addresses" }
+kaspa-addressmanager = { version = "0.17.1", path = "components/addressmanager" }
+kaspa-bip32 = { version = "0.17.1", path = "wallet/bip32" }
+kaspa-cli = { version = "0.17.1", path = "cli" }
+kaspa-connectionmanager = { version = "0.17.1", path = "components/connectionmanager" }
+kaspa-consensus = { version = "0.17.1", path = "consensus" }
+kaspa-consensus-core = { version = "0.17.1", path = "consensus/core" }
+kaspa-consensus-client = { version = "0.17.1", path = "consensus/client" }
+kaspa-consensus-notify = { version = "0.17.1", path = "consensus/notify" }
+kaspa-consensus-wasm = { version = "0.17.1", path = "consensus/wasm" }
+kaspa-consensusmanager = { version = "0.17.1", path = "components/consensusmanager" }
+kaspa-core = { version = "0.17.1", path = "core" }
+kaspa-daemon = { version = "0.17.1", path = "daemon" }
+kaspa-database = { version = "0.17.1", path = "database" }
+kaspa-grpc-client = { version = "0.17.1", path = "rpc/grpc/client" }
+kaspa-grpc-core = { version = "0.17.1", path = "rpc/grpc/core" }
+kaspa-grpc-server = { version = "0.17.1", path = "rpc/grpc/server" }
+kaspa-hashes = { version = "0.17.1", path = "crypto/hashes" }
+kaspa-index-core = { version = "0.17.1", path = "indexes/core" }
+kaspa-index-processor = { version = "0.17.1", path = "indexes/processor" }
+kaspa-math = { version = "0.17.1", path = "math" }
+kaspa-merkle = { version = "0.17.1", path = "crypto/merkle" }
+kaspa-metrics-core = { version = "0.17.1", path = "metrics/core" }
+kaspa-mining = { version = "0.17.1", path = "mining" }
+kaspa-mining-errors = { version = "0.17.1", path = "mining/errors" }
+kaspa-muhash = { version = "0.17.1", path = "crypto/muhash" }
+kaspa-notify = { version = "0.17.1", path = "notify" }
+kaspa-p2p-flows = { version = "0.17.1", path = "protocol/flows" }
+kaspa-p2p-lib = { version = "0.17.1", path = "protocol/p2p" }
+kaspa-perf-monitor = { version = "0.17.1", path = "metrics/perf_monitor" }
+kaspa-pow = { version = "0.17.1", path = "consensus/pow" }
+kaspa-rpc-core = { version = "0.17.1", path = "rpc/core" }
+kaspa-rpc-macros = { version = "0.17.1", path = "rpc/macros" }
+kaspa-rpc-service = { version = "0.17.1", path = "rpc/service" }
+kaspa-txscript = { version = "0.17.1", path = "crypto/txscript" }
+kaspa-txscript-errors = { version = "0.17.1", path = "crypto/txscript/errors" }
+kaspa-utils = { version = "0.17.1", path = "utils" }
+kaspa-utils-tower = { version = "0.17.1", path = "utils/tower" }
+kaspa-utxoindex = { version = "0.17.1", path = "indexes/utxoindex" }
+kaspa-wallet = { version = "0.17.1", path = "wallet/native" }
+kaspa-wallet-cli-wasm = { version = "0.17.1", path = "wallet/wasm" }
+kaspa-wallet-keys = { version = "0.17.1", path = "wallet/keys" }
+kaspa-wallet-pskt = { version = "0.17.1", path = "wallet/pskt" }
+kaspa-wallet-core = { version = "0.17.1", path = "wallet/core" }
+kaspa-wallet-macros = { version = "0.17.1", path = "wallet/macros" }
+kaspa-wasm = { version = "0.17.1", path = "wasm" }
+kaspa-wasm-core = { version = "0.17.1", path = "wasm/core" }
+kaspa-wrpc-client = { version = "0.17.1", path = "rpc/wrpc/client" }
+kaspa-wrpc-proxy = { version = "0.17.1", path = "rpc/wrpc/proxy" }
+kaspa-wrpc-server = { version = "0.17.1", path = "rpc/wrpc/server" }
+kaspa-wrpc-wasm = { version = "0.17.1", path = "rpc/wrpc/wasm" }
+kaspa-wrpc-example-subscriber = { version = "0.17.1", path = "rpc/wrpc/examples/subscriber" }
+kaspad = { version = "0.17.1", path = "kaspad" }
+kaspa-alloc = { version = "0.17.1", path = "utils/alloc" }
 
 # external
 aes = "0.8.3"

Cargo.toml

@@ -254,9 +254,9 @@ impl ConsensusSessionOwned {
         self.clone().spawn_blocking(move |c| c.get_current_block_color(hash)).await
     }
 
-    /// source refers to the earliest block from which the current node has full header & block data  
-    pub async fn async_get_source(&self) -> Hash {
-        self.clone().spawn_blocking(|c| c.get_source()).await
+    /// retention period root refers to the earliest block from which the current node has full header & block data  
+    pub async fn async_get_retention_period_root(&self) -> Hash {
+        self.clone().spawn_blocking(|c| c.get_retention_period_root()).await
     }
 
     pub async fn async_estimate_block_count(&self) -> BlockCount {

components/consensusmanager/src/session.rs

@@ -143,8 +143,8 @@ pub trait ConsensusApi: Send + Sync {
         unimplemented!()
     }
 
-    /// source refers to the earliest block from which the current node has full header & block data  
-    fn get_source(&self) -> Hash {
+    /// retention period root refers to the earliest block from which the current node has full header & block data  
+    fn get_retention_period_root(&self) -> Hash {
         unimplemented!()
     }
 

consensus/core/src/api/mod.rs

@@ -68,6 +68,9 @@ pub struct Config {
 
     /// A scale factor to apply to memory allocation bounds
     pub ram_scale: f64,
+
+    /// The number of days to keep data for
+    pub retention_period_days: Option<f64>,
 }
 
 impl Config {
@@ -95,6 +98,7 @@ impl Config {
             initial_utxo_set: Default::default(),
             disable_upnp: false,
             ram_scale: 1.0,
+            retention_period_days: None,
         }
     }
 

consensus/core/src/config/mod.rs

@@ -307,13 +307,39 @@ impl Consensus {
             is_consensus_exiting,
         };
 
-        // TODO (post HF): remove the upgrade
-        // Database upgrade to include pruning samples
-        this.pruning_samples_database_upgrade();
+        // Run database upgrades if any
+        this.run_database_upgrades();
 
         this
     }
 
+    /// A procedure for calling database upgrades which are self-contained (i.e., do not require knowing the DB version)
+    fn run_database_upgrades(&self) {
+        // Upgrade to initialize the new retention root field correctly
+        self.retention_root_database_upgrade();
+
+        // TODO (post HF): remove this upgrade
+        // Database upgrade to include pruning samples
+        self.pruning_samples_database_upgrade();
+    }
+
+    fn retention_root_database_upgrade(&self) {
+        let mut pruning_point_store = self.pruning_point_store.write();
+        if pruning_point_store.retention_period_root().unwrap_option().is_none() {
+            let mut batch = rocksdb::WriteBatch::default();
+            if self.config.is_archival {
+                // The retention checkpoint is what was previously known as history root
+                let retention_checkpoint = pruning_point_store.retention_checkpoint().unwrap();
+                pruning_point_store.set_retention_period_root(&mut batch, retention_checkpoint).unwrap();
+            } else {
+                // For non-archival nodes the retention root was the pruning point
+                let pruning_point = pruning_point_store.get().unwrap().pruning_point;
+                pruning_point_store.set_retention_period_root(&mut batch, pruning_point).unwrap();
+            }
+            self.db.write(batch).unwrap();
+        }
+    }
+
     fn pruning_samples_database_upgrade(&self) {
         //
         // For the first time this version runs, make sure we populate pruning samples
@@ -565,8 +591,8 @@ impl ConsensusApi for Consensus {
         // Verify the block exists and can be assumed to have relations and reachability data
         self.validate_block_exists(hash).ok()?;
 
-        // Verify that the block is in future(source), where Ghostdag data is complete
-        self.services.reachability_service.is_dag_ancestor_of(self.get_source(), hash).then_some(())?;
+        // Verify that the block is in future(retention root), where Ghostdag data is complete
+        self.services.reachability_service.is_dag_ancestor_of(self.get_retention_period_root(), hash).then_some(())?;
 
         let sink = self.get_sink();
 
@@ -621,31 +647,26 @@ impl ConsensusApi for Consensus {
         self.lkg_virtual_state.load().to_virtual_state_approx_id()
     }
 
-    fn get_source(&self) -> Hash {
-        if self.config.is_archival {
-            // we use the history root in archival cases.
-            return self.pruning_point_store.read().history_root().unwrap();
-        }
-        self.pruning_point_store.read().pruning_point().unwrap()
+    fn get_retention_period_root(&self) -> Hash {
+        self.pruning_point_store.read().retention_period_root().unwrap()
     }
 
-    /// Estimates number of blocks and headers stored in the node
+    /// Estimates the number of blocks and headers stored in the node database.
     ///
-    /// This is an estimation based on the daa score difference between the node's `source` and `sink`'s daa score,
+    /// This is an estimation based on the DAA score difference between the node's `retention root` and `virtual`'s DAA score,
     /// as such, it does not include non-daa blocks, and does not include headers stored as part of the pruning proof.  
     fn estimate_block_count(&self) -> BlockCount {
-        // PRUNE SAFETY: node is either archival or source is the pruning point which its header is kept permanently
-        let source_score = self.headers_store.get_compact_header_data(self.get_source()).unwrap().daa_score;
+        // PRUNE SAFETY: retention root is always a current or past pruning point which its header is kept permanently
+        let retention_period_root_score = self.headers_store.get_daa_score(self.get_retention_period_root()).unwrap();
         let virtual_score = self.get_virtual_daa_score();
         let header_count = self
             .headers_store
-            .get_compact_header_data(self.get_headers_selected_tip())
+            .get_daa_score(self.get_headers_selected_tip())
             .unwrap_option()
-            .map(|h| h.daa_score)
             .unwrap_or(virtual_score)
             .max(virtual_score)
-            - source_score;
-        let block_count = virtual_score - source_score;
+            - retention_period_root_score;
+        let block_count = virtual_score - retention_period_root_score;
         BlockCount { header_count, block_count }
     }
 
@@ -668,12 +689,12 @@ impl ConsensusApi for Consensus {
         // Verify that the block exists
         self.validate_block_exists(low)?;
 
-        // Verify that source is on chain(block)
+        // Verify that retention root is on chain(block)
         self.services
             .reachability_service
-            .is_chain_ancestor_of(self.get_source(), low)
+            .is_chain_ancestor_of(self.get_retention_period_root(), low)
             .then_some(())
-            .ok_or(ConsensusError::General("the queried hash does not have source on its chain"))?;
+            .ok_or(ConsensusError::General("the queried hash does not have retention root on its chain"))?;
 
         Ok(self.services.dag_traversal_manager.calculate_chain_path(low, self.get_sink(), chain_path_added_limit))
     }
@@ -748,7 +769,7 @@ impl ConsensusApi for Consensus {
     fn get_populated_transaction(&self, txid: Hash, accepting_block_daa_score: u64) -> Result<SignableTransaction, UtxoInquirerError> {
         // We need consistency between the pruning_point_store, utxo_diffs_store, block_transactions_store, selected chain and headers store reads
         let _guard = self.pruning_lock.blocking_read();
-        self.virtual_processor.get_populated_transaction(txid, accepting_block_daa_score, self.get_source())
+        self.virtual_processor.get_populated_transaction(txid, accepting_block_daa_score, self.get_retention_period_root())
     }
 
     fn get_virtual_parents(&self) -> BlockHashSet {

consensus/src/consensus/mod.rs

@@ -39,9 +39,16 @@ pub trait PruningStoreReader {
     fn get(&self) -> StoreResult<PruningPointInfo>;
 
     /// Represent the point after which data is fully held (i.e., history is consecutive from this point and up to virtual).
-    /// This is usually the pruning point, though it might lag a bit behind until data prune completes (and for archival
-    /// nodes it will remain the initial syncing point or the last pruning point before turning to an archive)
-    fn history_root(&self) -> StoreResult<Hash>;
+    /// This is usually a pruning point that is at or below the retention period requirement (and for archival
+    /// nodes it will remain the initial syncing point or the last pruning point before turning to an archive).
+    /// At every pruning point movement, this is adjusted to the next pruning point sample that satisfies the required
+    /// retention period.
+    fn retention_period_root(&self) -> StoreResult<Hash>;
+
+    // During pruning, this is a reference to the retention root before the pruning point move.
+    // After pruning, this is updated to point to the retention period root.
+    // This checkpoint is used to determine if pruning has successfully completed.
+    fn retention_checkpoint(&self) -> StoreResult<Hash>;
 }
 
 pub trait PruningStore: PruningStoreReader {
@@ -53,15 +60,17 @@ pub trait PruningStore: PruningStoreReader {
 pub struct DbPruningStore {
     db: Arc<DB>,
     access: CachedDbItem<PruningPointInfo>,
-    history_root_access: CachedDbItem<Hash>,
+    retention_checkpoint_access: CachedDbItem<Hash>,
+    retention_period_root_access: CachedDbItem<Hash>,
 }
 
 impl DbPruningStore {
     pub fn new(db: Arc<DB>) -> Self {
         Self {
             db: Arc::clone(&db),
             access: CachedDbItem::new(db.clone(), DatabaseStorePrefixes::PruningPoint.into()),
-            history_root_access: CachedDbItem::new(db, DatabaseStorePrefixes::HistoryRoot.into()),
+            retention_checkpoint_access: CachedDbItem::new(db.clone(), DatabaseStorePrefixes::RetentionCheckpoint.into()),
+            retention_period_root_access: CachedDbItem::new(db, DatabaseStorePrefixes::RetentionPeriodRoot.into()),
         }
     }
 
@@ -73,8 +82,12 @@ impl DbPruningStore {
         self.access.write(BatchDbWriter::new(batch), &PruningPointInfo { pruning_point, candidate, index })
     }
 
-    pub fn set_history_root(&mut self, batch: &mut WriteBatch, history_root: Hash) -> StoreResult<()> {
-        self.history_root_access.write(BatchDbWriter::new(batch), &history_root)
+    pub fn set_retention_checkpoint(&mut self, batch: &mut WriteBatch, retention_checkpoint: Hash) -> StoreResult<()> {
+        self.retention_checkpoint_access.write(BatchDbWriter::new(batch), &retention_checkpoint)
+    }
+
+    pub fn set_retention_period_root(&mut self, batch: &mut WriteBatch, retention_period_root: Hash) -> StoreResult<()> {
+        self.retention_period_root_access.write(BatchDbWriter::new(batch), &retention_period_root)
     }
 }
 
@@ -95,8 +108,12 @@ impl PruningStoreReader for DbPruningStore {
         self.access.read()
     }
 
-    fn history_root(&self) -> StoreResult<Hash> {
-        self.history_root_access.read()
+    fn retention_checkpoint(&self) -> StoreResult<Hash> {
+        self.retention_checkpoint_access.read()
+    }
+
+    fn retention_period_root(&self) -> StoreResult<Hash> {
+        self.retention_period_root_access.read()
     }
 }