Run Utreexo integration tests (keep-starknet-strange#229)

.gitignore

@@ -24,6 +24,8 @@ Cargo.lock
 .venv
 .python-version
 __pycache__
+.ipynb_checkpoints/
+*.ipynb
 
 .client_cache/
 .utxo_data/

packages/client/README.md

@@ -33,6 +33,12 @@ Run a specific test file (or several files):
 scarb test tests/data/light_481823.json
 ```
 
+Show output for a specific test:
+
+```sh
+scarb test tests/data/light_481823.json --nocapture
+```
+
 Re-generate integration test data:
 
 ```sh

packages/client/Scarb.toml

@@ -13,5 +13,4 @@ cairo_test.workspace = true
 test = "scarb build && ../../scripts/data/integration_tests.sh"
 regenerate_tests= "../../scripts/data/regenerate_tests.sh"
 client = "scarb build && ../../scripts/data/client.sh"
-utreexo = "scarb build && ../../scripts/data/utreexo.sh"
-lint = "flake8 scripts/ && black --check scripts/ && scarb fmt"
+lint = "black ../../scripts/data && flake8 ../../scripts/data --max-complexity=10 --max-line-length=88 && scarb fmt"

packages/client/src/main.cairo

@@ -1,14 +1,12 @@
 use consensus::types::block::Block;
-use consensus::types::state::State;
-use consensus::types::chain_state::BlockValidator;
-use consensus::types::utreexo::UtreexoStateTrait;
-use consensus::types::utxo_set::{UtxoSet, UtxoSetTrait};
+use consensus::types::chain_state::{ChainState, BlockValidator};
+use consensus::types::utxo_set::UtxoSet;
 
 /// Raito program arguments.
 #[derive(Serde)]
 struct Args {
-    /// Current (initial) state
-    state: State,
+    /// Current (initial) chain state
+    chain_state: ChainState,
     /// Batch of blocks that have to be applied to the current chain state
     blocks: Array<Block>,
 }
@@ -18,26 +16,18 @@ struct Args {
 /// Receives current state (chain state + utreexo state) and pending blocks,
 /// then validates and applies them one by one.
 /// Returns new state in case of success, otherwise raises an error.
-fn main(mut arguments: Span<felt252>) -> State {
-    let Args { mut state, blocks, } = Serde::deserialize(ref arguments)
+fn main(mut arguments: Span<felt252>) -> ChainState {
+    let Args { mut chain_state, blocks, } = Serde::deserialize(ref arguments)
         .expect('Failed to deserialize');
 
     let mut utxo_set: UtxoSet = Default::default();
 
     // Validate and apply block, accumulating UTXO updates in utxo_set
     for block in blocks {
-        state
-            .chain_state = state
-            .chain_state
+        chain_state = chain_state
             .validate_and_apply(block, ref utxo_set)
             .expect('Block validation failed');
     };
 
-    // Validate and apply UTXO updates
-    state.utreexo_state.validate_and_apply(ref utxo_set).expect('Utreexo validation failed');
-
-    // Ensure all UTXOs have been processed
-    utxo_set.finalize().expect('UtxoSet finalization failed');
-
-    state
+    chain_state
 }

packages/client/src/test.cairo

@@ -1,18 +1,32 @@
 use consensus::types::block::Block;
 use consensus::types::chain_state::{ChainState, BlockValidatorImpl};
-use consensus::types::state::{State};
 use consensus::types::utxo_set::{UtxoSet, UtxoSetTrait};
+use consensus::types::utreexo::{UtreexoState, UtreexoProof, UtreexoStateTrait};
 use core::testing::get_available_gas;
+use core::serde::Serde;
 
 /// Integration testing program arguments.
-#[derive(Serde)]
+#[derive(Drop)]
 struct Args {
     /// Current (initial) chain state
     chain_state: ChainState,
     /// Batch of blocks that have to be applied to the current chain state
     blocks: Array<Block>,
     /// Expected chain state (that we want to compare the result with)
     expected_chain_state: ChainState,
+    /// Optional Utreexo arguments
+    utreexo_args: Option<UtreexoArgs>,
+}
+
+/// Utreexo arguments necessary for constraining the UTXO set
+#[derive(Drop, Serde)]
+struct UtreexoArgs {
+    /// Current (initial) accumulator state
+    state: UtreexoState,
+    /// Inclusion proofs for TXOs spent during program run
+    proofs: Array<UtreexoProof>,
+    /// Expected accumulator state at the end of the execution
+    expected_state: UtreexoState,
 }
 
 /// Integration testing program entrypoint.
@@ -21,21 +35,19 @@ struct Args {
 /// Panics in case of a validation error or chain state mismatch.
 /// Prints result to the stdout.
 fn test(mut arguments: Span<felt252>) {
-    let Args { mut chain_state, blocks, expected_chain_state } = Serde::deserialize(ref arguments)
+    let Args { mut chain_state, blocks, expected_chain_state, utreexo_args } = Serde::deserialize(
+        ref arguments
+    )
         .expect('Failed to deserialize');
 
-    // Temporary solution while script doesn't handle utreexo.
-    // Allows to test one isolated block, or a batch of blocks starting from genesis.
-    let mut state: State = State { chain_state: chain_state, utreexo_state: Default::default(), };
     let mut utxo_set: UtxoSet = Default::default();
-
     let mut gas_before = get_available_gas();
 
     for block in blocks {
-        let height = state.chain_state.block_height + 1;
-        match state.chain_state.validate_and_apply(block, ref utxo_set) {
+        let height = chain_state.block_height + 1;
+        match chain_state.validate_and_apply(block, ref utxo_set) {
             Result::Ok(new_chain_state) => {
-                state.chain_state = new_chain_state;
+                chain_state = new_chain_state;
                 let gas_after = get_available_gas();
                 println!("OK: block={} gas_spent={}", height, gas_before - gas_after);
                 gas_before = gas_after;
@@ -50,43 +62,60 @@ fn test(mut arguments: Span<felt252>) {
         }
     };
 
-    if state.chain_state != expected_chain_state {
+    if chain_state != expected_chain_state {
         println!(
             "FAIL: block={} error='expected chain state {:?}, actual {:?}'",
-            state.chain_state.block_height,
+            chain_state.block_height,
             expected_chain_state,
-            state.chain_state
+            chain_state
         );
         panic!();
     }
 
-    // TODO: provide the expected utreexo state via args and compare it with the actual one
-    //
-    // gas_before = get_available_gas();
-    //
-    // match state.utreexo_state.validate_and_apply(ref utxo_set) {
-    //     Result::Ok(()) => {
-    //         let gas_after = get_available_gas();
-    //         println!("OK: gas_spent={}", gas_before - gas_after);
-    //     },
-    //     Result::Err(err) => {
-    //         let gas_after = get_available_gas();
-    //         println!("FAIL: gas_spent={} error='{:?}'", gas_before - gas_after, err);
-    //         panic!();
-    //     }
-    // }
-    //
-    // if state.utreexo_state != expected_utreexo_state {
-    //     println!(
-    //         "FAIL: error='expected utreexo state {:?}, actual {:?}'",
-    //         expected_utreexo_state,
-    //         state.utreexo_state
-    //     );
-    //     panic!();
-    // }
-
     if let Result::Err(err) = utxo_set.finalize() {
         println!("FAIL: error='{}'", err);
         panic!();
     }
+
+    if let Option::Some(UtreexoArgs { mut state, proofs, expected_state }) = utreexo_args {
+        match state.validate_and_apply(utxo_set, proofs.span()) {
+            Result::Ok(()) => {
+                let gas_after = get_available_gas();
+                println!("OK: gas_spent={}", gas_before - gas_after);
+            },
+            Result::Err(err) => {
+                let gas_after = get_available_gas();
+                println!("FAIL: gas_spent={} error='{:?}'", gas_before - gas_after, err);
+                panic!();
+            }
+        }
+
+        if state != expected_state {
+            println!(
+                "FAIL: error='expected utreexo state {:?}, actual {:?}'", expected_state, state
+            );
+            panic!();
+        }
+    }
+}
+
+/// Workaround for handling missing `utreexo_args` field.
+/// Rough analogue of `#[serde(default)]`
+impl ArgsSerde of Serde<Args> {
+    fn serialize(self: @Args, ref output: Array<felt252>) {
+        panic!("not implemented");
+    }
+
+    fn deserialize(ref serialized: Span<felt252>) -> Option<Args> {
+        let chain_state: ChainState = Serde::deserialize(ref serialized).expect('chain_state');
+        let blocks: Array<Block> = Serde::deserialize(ref serialized).expect('blocks');
+        let expected_chain_state: ChainState = Serde::deserialize(ref serialized)
+            .expect('expected_chain_state');
+        let utreexo_args: Option<UtreexoArgs> = if serialized.len() > 0 {
+            Option::Some(Serde::deserialize(ref serialized).expect('utreexo_args'))
+        } else {
+            Option::None
+        };
+        Option::Some(Args { chain_state, blocks, expected_chain_state, utreexo_args, })
+    }
 }