From f0a7ab2d18b508cca1708976e5ee1a4f6a63a9d3 Mon Sep 17 00:00:00 2001 From: Sai <135601871+sai-deng@users.noreply.github.com> Date: Tue, 15 Oct 2024 22:20:34 +0000 Subject: [PATCH 01/17] Optimize zkVM Proving by Skipping Unused Keccak Tables (#690) This PR introduces optional Keccak table proving, which optimizes performance by skipping the proving process when the Keccak tables are empty. This change reduces unnecessary computation and requires an update to Plonky2 for full integration. --- Cargo.lock | 14 +- Cargo.toml | 6 +- .../src/fixed_recursive_verifier.rs | 247 +++++++++--------- evm_arithmetization/src/get_challenges.rs | 29 +- evm_arithmetization/src/proof.rs | 27 +- evm_arithmetization/src/prover.rs | 42 +-- evm_arithmetization/src/testing_utils.rs | 27 +- evm_arithmetization/src/verifier.rs | 73 ++++-- evm_arithmetization/tests/empty_tables.rs | 80 ++++++ zero/src/prover_state/mod.rs | 55 ++-- 10 files changed, 373 insertions(+), 227 deletions(-) create mode 100644 evm_arithmetization/tests/empty_tables.rs diff --git a/Cargo.lock b/Cargo.lock index 68f584f54..9cd9634ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3633,7 +3633,7 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plonky2" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" dependencies = [ "ahash", "anyhow", @@ -3644,7 +3644,7 @@ dependencies = [ "log", "num", "plonky2_field", - "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed)", + "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee)", "plonky2_util", "rand", "rand_chacha", @@ -3657,7 +3657,7 @@ dependencies = [ [[package]] name = "plonky2_field" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" dependencies = [ "anyhow", "itertools 0.11.0", @@ -3681,7 +3681,7 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" dependencies = [ "rayon", ] @@ -3689,7 +3689,7 @@ dependencies = [ [[package]] name = "plonky2_util" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" [[package]] name = "plotters" @@ -4650,7 +4650,7 @@ checksum = "8acdd7dbfcfb5dd6e46c63512508bf71c2043f70b8f143813ad75cb5e8a589f2" [[package]] name = "starky" version = "0.4.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" dependencies = [ "ahash", "anyhow", @@ -4659,7 +4659,7 @@ dependencies = [ "log", "num-bigint", "plonky2", - "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed)", + "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee)", "plonky2_util", ] diff --git a/Cargo.toml b/Cargo.toml index 0bd15a3bd..1f952c7b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -114,10 +114,10 @@ zk_evm_proc_macro = { path = "proc_macro", version = "0.1.0" } zero = { path = "zero", default-features = false } # plonky2-related dependencies -plonky2 = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f2b06500e16ad4d7f1c2b057903602eed" } +plonky2 = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } plonky2_maybe_rayon = "0.2.0" -plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f2b06500e16ad4d7f1c2b057903602eed" } -starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f2b06500e16ad4d7f1c2b057903602eed" } +plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } +starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } [workspace.lints.clippy] too_long_first_doc_paragraph = "allow" diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 0d7bb82dd..96ecce6d3 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -131,6 +131,8 @@ where /// Holds chains of circuits for each table and for each initial /// `degree_bits`. pub by_table: [RecursiveCircuitsForTable; NUM_TABLES], + /// Dummy proofs of each table for the root circuit. + pub table_dummy_proofs: [Option>; NUM_TABLES], } /// Data for the EVM root circuit, which is used to combine each STARK's shrunk @@ -569,6 +571,43 @@ where } } +/// A struct that encapsulates both the init degree and the shrunk proof. +#[derive(Eq, PartialEq, Debug)] +pub struct ShrunkProofData, C: GenericConfig, const D: usize> +{ + /// The [`CommonCircuitData`] of the last shrinking circuit. + pub common_circuit_data: CommonCircuitData, + + /// The proof after applying shrinking recursion. + pub proof: ProofWithPublicInputs, +} + +impl, C: GenericConfig, const D: usize> + ShrunkProofData +{ + fn to_buffer( + &self, + buffer: &mut Vec, + gate_serializer: &dyn GateSerializer, + ) -> IoResult<()> { + buffer.write_common_circuit_data(&self.common_circuit_data, gate_serializer)?; + buffer.write_proof_with_public_inputs(&self.proof)?; + Ok(()) + } + + fn from_buffer( + buffer: &mut Buffer, + gate_serializer: &dyn GateSerializer, + ) -> IoResult { + let common_circuit_data = buffer.read_common_circuit_data(gate_serializer)?; + let proof = buffer.read_proof_with_public_inputs(&common_circuit_data)?; + Ok(Self { + common_circuit_data, + proof, + }) + } +} + impl AllRecursiveCircuits where F: RichField + Extendable, @@ -612,6 +651,15 @@ where table.to_buffer(&mut buffer, gate_serializer, generator_serializer)?; } } + for table in &self.table_dummy_proofs { + match table { + Some(dummy_proof_data) => { + buffer.write_bool(true)?; + dummy_proof_data.to_buffer(&mut buffer, gate_serializer)? + } + None => buffer.write_bool(false)?, + } + } Ok(buffer) } @@ -690,6 +738,14 @@ where } }; + let table_dummy_proofs = core::array::from_fn(|_| { + if buffer.read_bool().ok()? { + Some(ShrunkProofData::from_buffer(&mut buffer, gate_serializer).ok()?) + } else { + None + } + }); + Ok(Self { root, segment_aggregation, @@ -698,6 +754,7 @@ where block_wrapper, two_to_one_block, by_table, + table_dummy_proofs, }) } @@ -774,6 +831,33 @@ where let block_wrapper = Self::create_block_wrapper_circuit(&block); let two_to_one_block = Self::create_two_to_one_block_circuit(&block_wrapper); + // TODO(sdeng): enable more optional Tables + let table_dummy_proofs = core::array::from_fn(|i| { + if KECCAK_TABLES_INDICES.contains(&i) { + let init_degree = degree_bits_ranges[i].start; + let common_circuit_data = by_table[i] + .by_stark_size + .get(&init_degree) + .expect("Unable to get the shrinking circuits") + .shrinking_wrappers + .last() + .expect("Unable to get the last shrinking circuit") + .circuit + .common + .clone(); + let dummy_circuit: CircuitData = dummy_circuit(&common_circuit_data); + let dummy_pis = HashMap::new(); + let proof = dummy_proof(&dummy_circuit, dummy_pis) + .expect("Unable to generate dummy proofs"); + Some(ShrunkProofData { + common_circuit_data, + proof, + }) + } else { + None + } + }); + Self { root, segment_aggregation, @@ -782,6 +866,7 @@ where block_wrapper, two_to_one_block, by_table, + table_dummy_proofs, } } @@ -1903,7 +1988,6 @@ where abort_signal: Option>, ) -> anyhow::Result> { features_check(&generation_inputs); - let all_proof = prove::( all_stark, config, @@ -1912,37 +1996,32 @@ where timing, abort_signal.clone(), )?; + self.prove_segment_with_all_proofs(&all_proof, config, abort_signal.clone()) + } + pub fn prove_segment_with_all_proofs( + &self, + all_proof: &AllProof, + config: &StarkConfig, + abort_signal: Option>, + ) -> anyhow::Result> { let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { let table_circuits = &self.by_table[table]; if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { - // generate and set a dummy `index_verifier_data` and `proof_with_pis` - let index_verifier_data = - table_circuits.by_stark_size.keys().min().ok_or_else(|| { - anyhow::format_err!("No valid size in shrinking circuits") - })?; - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_usize(*index_verifier_data), + let dummy_proof_data = self.table_dummy_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("No dummy_proof_data"))?; + root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO); + root_inputs.set_proof_with_pis_target( + &self.root.proof_with_pis[table], + &dummy_proof_data.proof, ); - let table_circuit = table_circuits - .by_stark_size - .get(index_verifier_data) - .ok_or_else(|| anyhow::format_err!("No valid size in shrinking circuits"))? - .shrinking_wrappers - .last() - .ok_or_else(|| anyhow::format_err!("No shrinking circuits"))?; - let dummy_circuit: CircuitData = - dummy_circuit(&table_circuit.circuit.common); - let dummy_pis = HashMap::new(); - let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) - .expect("Unable to generate dummy proofs"); - root_inputs - .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); } else { - let stark_proof = &all_proof.multi_proof.stark_proofs[table]; + let stark_proof = &all_proof.multi_proof.stark_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("Unable to get stark proof"))?; let original_degree_bits = stark_proof.proof.recover_degree_bits(config); let shrunk_proof = table_circuits .by_stark_size @@ -1993,7 +2072,7 @@ where is_agg: false, is_dummy: false, proof_with_pvs: ProofWithPublicValues { - public_values: all_proof.public_values, + public_values: all_proof.public_values.clone(), intern: root_proof, }, }) @@ -2052,40 +2131,34 @@ where pub fn prove_segment_after_initial_stark( &self, all_proof: AllProof, - table_circuits: &[(RecursiveCircuitsForTableSize, u8); NUM_TABLES], + table_circuits: &[Option<(RecursiveCircuitsForTableSize, u8)>; NUM_TABLES], abort_signal: Option>, ) -> anyhow::Result> { let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { - let (table_circuit, index_verifier_data) = &table_circuits[table]; if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_u8(*index_verifier_data), + let dummy_proof = self.table_dummy_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("Unable to get dummpy proof"))?; + root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO); + root_inputs.set_proof_with_pis_target( + &self.root.proof_with_pis[table], + &dummy_proof.proof, ); - // generate and set a dummy `proof_with_pis` - let common_data = &table_circuit - .shrinking_wrappers - .last() - .ok_or_else(|| anyhow::format_err!("No shrinking circuits"))? - .circuit - .common; - let dummy_circuit: CircuitData = dummy_circuit(common_data); - let dummy_pis = HashMap::new(); - let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) - .expect("Unable to generate dummy proofs"); - root_inputs - .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); } else { - let stark_proof = &all_proof.multi_proof.stark_proofs[table]; - - let shrunk_proof = - table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; + let (table_circuit, index_verifier_data) = &table_circuits[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("Unable to get circuits"))?; root_inputs.set_target( self.root.index_verifier_data[table], F::from_canonical_u8(*index_verifier_data), ); + let stark_proof = all_proof.multi_proof.stark_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("Unable to get stark proof"))?; + let shrunk_proof = + table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; root_inputs .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); } @@ -2736,6 +2809,7 @@ where agg_inputs.set_proof_with_pis_target(&agg_child.base_proof, proof); } } + /// A map between initial degree sizes and their associated shrinking recursion /// circuits. #[derive(Eq, PartialEq, Debug)] @@ -2767,6 +2841,7 @@ where buffer.write_usize(size)?; table.to_buffer(buffer, gate_serializer, generator_serializer)?; } + Ok(()) } @@ -2786,6 +2861,7 @@ where )?; by_stark_size.insert(key, table); } + Ok(Self { by_stark_size }) } @@ -2810,6 +2886,7 @@ where ) }) .collect(); + Self { by_stark_size } } @@ -3131,79 +3208,3 @@ pub mod testing { } } } - -#[cfg(test)] -#[cfg(not(feature = "cdk_erigon"))] -mod tests { - use plonky2::field::goldilocks_field::GoldilocksField; - use plonky2::plonk::config::PoseidonGoldilocksConfig; - use plonky2::timed; - - use super::*; - use crate::testing_utils::{empty_payload, init_logger}; - use crate::witness::operation::Operation; - - type F = GoldilocksField; - const D: usize = 2; - type C = PoseidonGoldilocksConfig; - - #[test] - fn test_segment_proof_generation_without_keccak() -> anyhow::Result<()> { - init_logger(); - - let all_stark = AllStark::::default(); - let config = StarkConfig::standard_fast_config(); - - // Generate a dummy payload for testing - let payload = empty_payload()?; - let max_cpu_len_log = Some(7); - let mut segment_iterator = SegmentDataIterator::::new(&payload, max_cpu_len_log); - let (_, mut segment_data) = segment_iterator.next().unwrap()?; - - let opcode_counts = &segment_data.opcode_counts; - assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); - - let timing = &mut TimingTree::new( - "Segment Proof Generation Without Keccak Test", - log::Level::Info, - ); - // Process and prove segment - let all_circuits = timed!( - timing, - log::Level::Info, - "Create all recursive circuits", - AllRecursiveCircuits::::new( - &all_stark, - &[16..17, 8..9, 7..8, 4..9, 8..9, 4..7, 16..17, 16..17, 16..17], - &config, - ) - ); - - let segment_proof = timed!( - timing, - log::Level::Info, - "Prove segment", - all_circuits.prove_segment( - &all_stark, - &config, - payload.trim(), - &mut segment_data, - timing, - None, - )? - ); - - // Verify the generated segment proof - timed!( - timing, - log::Level::Info, - "Verify segment proof", - all_circuits.verify_root(segment_proof.proof_with_pvs.intern.clone())? - ); - - // Print timing details - timing.print(); - - Ok(()) - } -} diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index ca1f06c62..c7471471b 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -247,7 +247,7 @@ pub(crate) fn observe_public_values_target< pub mod testing { use plonky2::field::extension::Extendable; - use plonky2::hash::hash_types::RichField; + use plonky2::hash::hash_types::{RichField, NUM_HASH_OUT_ELTS}; use plonky2::iop::challenger::Challenger; use plonky2::plonk::config::GenericConfig; use starky::config::StarkConfig; @@ -278,19 +278,14 @@ pub mod testing { let stark_proofs = &self.multi_proof.stark_proofs; - for (i, proof) in stark_proofs.iter().enumerate() { - if KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables { - // Observe zero merkle caps when skipping Keccak tables. - let zero_merkle_cap = proof - .proof - .trace_cap - .flatten() - .iter() - .map(|_| F::ZERO) - .collect::>(); - challenger.observe_elements(&zero_merkle_cap); + for (i, stark_proof) in stark_proofs.iter().enumerate() { + if let Some(stark_proof) = stark_proof { + challenger.observe_cap(&stark_proof.proof.trace_cap); } else { - challenger.observe_cap(&proof.proof.trace_cap); + assert!(KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables); + let zero_cap = + vec![F::ZERO; config.fri_config.num_cap_elements() * NUM_HASH_OUT_ELTS]; + challenger.observe_elements(&zero_cap); } } @@ -301,16 +296,16 @@ pub mod testing { Ok(AllProofChallenges { stark_challenges: core::array::from_fn(|i| { - if KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables { - None - } else { + if let Some(stark_proof) = &stark_proofs[i] { challenger.compact(); - Some(stark_proofs[i].proof.get_challenges( + Some(stark_proof.proof.get_challenges( &mut challenger, Some(&ctl_challenges), true, config, )) + } else { + None } }), ctl_challenges, diff --git a/evm_arithmetization/src/proof.rs b/evm_arithmetization/src/proof.rs index 304072aed..b0de028af 100644 --- a/evm_arithmetization/src/proof.rs +++ b/evm_arithmetization/src/proof.rs @@ -10,7 +10,8 @@ use plonky2::plonk::config::{GenericConfig, GenericHashOut, Hasher}; use plonky2::util::serialization::{Buffer, IoResult, Read, Write}; use serde::{Deserialize, Serialize}; use starky::config::StarkConfig; -use starky::proof::MultiProof; +use starky::lookup::GrandProductChallengeSet; +use starky::proof::StarkProofWithMetadata; use crate::all_stark::NUM_TABLES; use crate::util::{get_h160, get_h256, get_u256, h256_limbs, h2u}; @@ -21,6 +22,22 @@ pub(crate) const DEFAULT_CAP_HEIGHT: usize = 4; /// Number of elements contained in a Merkle cap with default height. pub(crate) const DEFAULT_CAP_LEN: usize = 1 << DEFAULT_CAP_HEIGHT; +/// A combination of STARK proofs for independent statements operating on +/// possibly shared variables, along with Cross-Table Lookup (CTL) challenges to +/// assert consistency of common variables across tables. +#[derive(Debug, Clone)] +pub struct MultiProof< + F: RichField + Extendable, + C: GenericConfig, + const D: usize, + const N: usize, +> { + /// Proofs for all the different STARK modules. + pub stark_proofs: [Option>; N], + /// Cross-table lookup challenges. + pub ctl_challenges: GrandProductChallengeSet, +} + /// A STARK proof for each table, plus some metadata used to create recursive /// wrapper proofs. #[derive(Debug, Clone)] @@ -38,8 +55,12 @@ pub struct AllProof, C: GenericConfig, co impl, C: GenericConfig, const D: usize> AllProof { /// Returns the degree (i.e. the trace length) of each STARK. - pub fn degree_bits(&self, config: &StarkConfig) -> [usize; NUM_TABLES] { - self.multi_proof.recover_degree_bits(config) + pub fn degree_bits(&self, config: &StarkConfig) -> [Option; NUM_TABLES] { + core::array::from_fn(|i| { + self.multi_proof.stark_proofs[i] + .as_ref() + .map(|proof| proof.proof.recover_degree_bits(config)) + }) } } diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index fdc5eca20..4192db6c8 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -16,7 +16,7 @@ use plonky2::util::timing::TimingTree; use starky::config::StarkConfig; use starky::cross_table_lookup::{get_ctl_data, CtlData}; use starky::lookup::GrandProductChallengeSet; -use starky::proof::{MultiProof, StarkProofWithMetadata}; +use starky::proof::StarkProofWithMetadata; use starky::prover::prove_with_commitment; use starky::stark::Stark; @@ -24,7 +24,7 @@ use crate::all_stark::{AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES}; use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::{generate_traces, GenerationInputs, TrimmedGenerationInputs}; use crate::get_challenges::observe_public_values; -use crate::proof::{AllProof, MemCap, PublicValues, DEFAULT_CAP_LEN}; +use crate::proof::{AllProof, MemCap, MultiProof, PublicValues, DEFAULT_CAP_LEN}; use crate::GenerationSegmentData; /// Generate traces, then create all STARK proofs. @@ -220,7 +220,7 @@ where } type ProofWithMemCaps = ( - [StarkProofWithMetadata; NUM_TABLES], + [Option>; NUM_TABLES], MerkleCap, MerkleCap, ); @@ -273,16 +273,16 @@ where let (arithmetic_proof, _) = prove_table!(arithmetic_stark, Table::Arithmetic); let (byte_packing_proof, _) = prove_table!(byte_packing_stark, Table::BytePacking); let (cpu_proof, _) = prove_table!(cpu_stark, Table::Cpu); - let challenger_after_cpu = challenger.clone(); - // TODO(sdeng): Keccak proofs are still required for CTLs, etc. Refactor the - // code and remove the unnecessary parts. - let (keccak_proof, _) = prove_table!(keccak_stark, Table::Keccak); - let (keccak_sponge_proof, _) = prove_table!(keccak_sponge_stark, Table::KeccakSponge); - if !use_keccak_tables { - // We need to connect the challenger state of CPU and Logic tables when the - // Keccak tables are not in use. - *challenger = challenger_after_cpu; - } + let keccak_proof = if use_keccak_tables { + Some(prove_table!(keccak_stark, Table::Keccak).0) + } else { + None + }; + let keccak_sponge_proof = if use_keccak_tables { + Some(prove_table!(keccak_sponge_stark, Table::KeccakSponge).0) + } else { + None + }; let (logic_proof, _) = prove_table!(logic_stark, Table::Logic); let (memory_proof, _) = prove_table!(memory_stark, Table::Memory); let (mem_before_proof, mem_before_cap) = prove_table!(mem_before_stark, Table::MemBefore); @@ -293,17 +293,17 @@ where Ok(( [ - arithmetic_proof, - byte_packing_proof, - cpu_proof, + Some(arithmetic_proof), + Some(byte_packing_proof), + Some(cpu_proof), keccak_proof, keccak_sponge_proof, - logic_proof, - memory_proof, - mem_before_proof, - mem_after_proof, + Some(logic_proof), + Some(memory_proof), + Some(mem_before_proof), + Some(mem_after_proof), #[cfg(feature = "cdk_erigon")] - poseidon_proof, + Some(poseidon_proof), ], mem_before_cap, mem_after_cap, diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 5f1b1b041..039dc2504 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -10,12 +10,17 @@ use mpt_trie::{ nibbles::Nibbles, partial_trie::{HashedPartialTrie, Node, PartialTrie}, }; +use plonky2::field::goldilocks_field::GoldilocksField; pub use crate::cpu::kernel::cancun_constants::*; pub use crate::cpu::kernel::constants::global_exit_root::*; -use crate::generation::TrieInputs; +use crate::generation::{TrieInputs, TrimmedGenerationInputs}; use crate::proof::TrieRoots; -use crate::{generation::mpt::AccountRlp, proof::BlockMetadata, util::h2u, GenerationInputs}; +use crate::witness::operation::Operation; +use crate::{ + generation::mpt::AccountRlp, proof::BlockMetadata, util::h2u, GenerationInputs, + GenerationSegmentData, SegmentDataIterator, +}; pub const EMPTY_NODE_HASH: H256 = H256(hex!( "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" @@ -165,7 +170,7 @@ pub fn scalable_contract_from_storage(storage_trie: &HashedPartialTrie) -> Accou } } -pub fn empty_payload() -> Result { +fn empty_payload() -> Result { // Set up default block metadata let block_metadata = BlockMetadata { block_beneficiary: Address::zero(), @@ -213,3 +218,19 @@ pub fn empty_payload() -> Result { Ok(inputs) } + +pub fn segment_with_empty_tables() -> Result<( + TrimmedGenerationInputs, + GenerationSegmentData, +)> { + let payload = empty_payload()?; + let max_cpu_len_log = Some(7); + let mut segment_iterator = + SegmentDataIterator::::new(&payload, max_cpu_len_log); + let (trimmed_inputs, segment_data) = segment_iterator.next().unwrap()?; + + let opcode_counts = &segment_data.opcode_counts; + assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); + + Ok((trimmed_inputs, segment_data)) +} diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 947516801..81e9b502b 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -85,7 +85,9 @@ pub mod testing { use plonky2::hash::hash_types::RichField; use plonky2::plonk::config::{GenericConfig, GenericHashOut}; use starky::config::StarkConfig; - use starky::cross_table_lookup::{get_ctl_vars_from_proofs, verify_cross_table_lookups}; + use starky::cross_table_lookup::verify_cross_table_lookups; + use starky::cross_table_lookup::CrossTableLookup; + use starky::cross_table_lookup::CtlCheckVars; use starky::lookup::GrandProductChallenge; use starky::stark::Stark; use starky::verifier::verify_stark_proof_with_challenges; @@ -184,25 +186,39 @@ pub mod testing { cross_table_lookups, } = all_stark; - let ctl_vars_per_table = get_ctl_vars_from_proofs( - &all_proof.multi_proof, - cross_table_lookups, - &ctl_challenges, - &num_lookup_columns, - all_stark.arithmetic_stark.constraint_degree(), - ); - let stark_proofs = &all_proof.multi_proof.stark_proofs; macro_rules! verify_table { ($stark:ident, $table:expr) => { + let stark_proof = &stark_proofs[*$table] + .as_ref() + .expect("Missing stark_proof") + .proof; + let ctl_vars = { + let (total_num_helpers, _, num_helpers_by_ctl) = + CrossTableLookup::num_ctl_helpers_zs_all( + &all_stark.cross_table_lookups, + *$table, + config.num_challenges, + $stark.constraint_degree(), + ); + CtlCheckVars::from_proof( + *$table, + &stark_proof, + &all_stark.cross_table_lookups, + &ctl_challenges, + num_lookup_columns[*$table], + total_num_helpers, + &num_helpers_by_ctl, + ) + }; verify_stark_proof_with_challenges( $stark, - &stark_proofs[*$table].proof, + stark_proof, &stark_challenges[*$table] .as_ref() .expect("Missing challenges"), - Some(&ctl_vars_per_table[*$table]), + Some(&ctl_vars), &[], config, )?; @@ -240,12 +256,39 @@ pub mod testing { .map(|i| get_memory_extra_looking_sum(&public_values, ctl_challenges.challenges[i])) .collect_vec(); + let all_ctls = &all_stark.cross_table_lookups; + verify_cross_table_lookups::( cross_table_lookups, - all_proof - .multi_proof - .stark_proofs - .map(|p| p.proof.openings.ctl_zs_first.unwrap()), + core::array::from_fn(|i| { + if let Some(stark_proof) = &stark_proofs[i] { + stark_proof + .proof + .openings + .ctl_zs_first + .as_ref() + .expect("Missing ctl_zs") + .clone() + } else if i == *Table::Keccak { + let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( + all_ctls, + *Table::Keccak, + config.num_challenges, + keccak_stark.constraint_degree(), + ); + vec![F::ZERO; n] + } else if i == *Table::KeccakSponge { + let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( + all_ctls, + *Table::KeccakSponge, + config.num_challenges, + keccak_sponge_stark.constraint_degree(), + ); + vec![F::ZERO; n] + } else { + panic!("Unable to find stark_proof"); + } + }), Some(&extra_looking_sums), config, ) diff --git a/evm_arithmetization/tests/empty_tables.rs b/evm_arithmetization/tests/empty_tables.rs new file mode 100644 index 000000000..d25901e24 --- /dev/null +++ b/evm_arithmetization/tests/empty_tables.rs @@ -0,0 +1,80 @@ +#![cfg(feature = "eth_mainnet")] + +use evm_arithmetization::fixed_recursive_verifier::AllRecursiveCircuits; +use evm_arithmetization::prover::prove; +use evm_arithmetization::testing_utils::{init_logger, segment_with_empty_tables}; +use evm_arithmetization::verifier::testing::verify_all_proofs; +use evm_arithmetization::AllStark; +use plonky2::field::goldilocks_field::GoldilocksField; +use plonky2::plonk::config::PoseidonGoldilocksConfig; +use plonky2::timed; +use plonky2::util::timing::TimingTree; +use starky::config::StarkConfig; + +/// This test focuses on testing zkVM proofs with some empty tables. +#[test] +fn empty_tables() -> anyhow::Result<()> { + type F = GoldilocksField; + const D: usize = 2; + type C = PoseidonGoldilocksConfig; + + init_logger(); + + let all_stark = AllStark::::default(); + let config = StarkConfig::standard_fast_config(); + let timing = &mut TimingTree::new("Empty Table Test", log::Level::Info); + + // Generate segment data + let (payload, mut segment_data) = segment_with_empty_tables()?; + + // Create all STARK proofs + let mut proofs = vec![]; + let proof = timed!( + timing, + log::Level::Info, + "Create all STARK proofs", + prove::( + &all_stark, + &config, + payload, + &mut segment_data, + timing, + None, + )? + ); + proofs.push(proof); + + // Verify the generated STARK proofs + verify_all_proofs(&all_stark, &proofs, &config)?; + + // Process and generate segment proof + let all_circuits = timed!( + timing, + log::Level::Info, + "Create all recursive circuits", + AllRecursiveCircuits::::new( + &all_stark, + &[16..17, 8..9, 7..8, 4..6, 8..9, 4..5, 16..17, 16..17, 16..17], + &config, + ) + ); + let segment_proof = timed!( + timing, + log::Level::Info, + "Prove segment", + all_circuits.prove_segment_with_all_proofs(&proofs[0], &config, None)? + ); + + // Verify the generated segment proof + timed!( + timing, + log::Level::Info, + "Verify segment proof", + all_circuits.verify_root(segment_proof.proof_with_pvs.intern.clone())? + ); + + // Print timing details + timing.print(); + + Ok(()) +} diff --git a/zero/src/prover_state/mod.rs b/zero/src/prover_state/mod.rs index edbad02ba..8cdfb45ce 100644 --- a/zero/src/prover_state/mod.rs +++ b/zero/src/prover_state/mod.rs @@ -174,45 +174,30 @@ impl ProverStateManager { &self, config: &StarkConfig, all_proof: &AllProof, - ) -> anyhow::Result<[(RecursiveCircuitsForTableSize, u8); NUM_TABLES]> { + ) -> anyhow::Result<[Option<(RecursiveCircuitsForTableSize, u8)>; NUM_TABLES]> { let degrees = all_proof.degree_bits(config); - /// Given a recursive circuit index (e.g., Arithmetic / 0), return a - /// tuple containing the loaded table at the specified size and - /// its offset relative to the configured range used to pre-process the - /// circuits. - macro_rules! circuit { - ($circuit_index:expr) => { - ( - RecursiveCircuitResource::get(&( - $circuit_index.into(), - degrees[$circuit_index], + // Given a recursive circuit index (e.g., Arithmetic / 0), return a + // tuple containing the loaded table at the specified size and + // its offset relative to the configured range used to pre-process the + // circuits. + let circuits = core::array::from_fn(|i| match degrees[i] { + Some(size) => RecursiveCircuitResource::get(&(i.into(), size)) + .map(|circuit_resource| { + Some(( + circuit_resource, + (size - self.circuit_config[i].start) as u8, )) - .map_err(|e| { - let circuit: $crate::prover_state::circuit::Circuit = $circuit_index.into(); - let size = degrees[$circuit_index]; - anyhow::Error::from(e).context(format!( - "Attempting to load circuit: {circuit:?} at size: {size}" - )) - })?, - (degrees[$circuit_index] - self.circuit_config[$circuit_index].start) as u8, - ) - }; - } + }) + .map_err(|e| { + anyhow::Error::from(e) + .context(format!("Attempting to load circuit: {i} at size: {size}")) + }) + .unwrap_or(None), + None => None, + }); - Ok([ - circuit!(0), - circuit!(1), - circuit!(2), - circuit!(3), - circuit!(4), - circuit!(5), - circuit!(6), - circuit!(7), - circuit!(8), - #[cfg(feature = "cdk_erigon")] - circuit!(9), - ]) + Ok(circuits) } /// Generate a segment proof using the specified input, loading From 0ec9970fbb8164a8221ffcaec9700b1db5df6080 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Wed, 16 Oct 2024 14:43:14 +0530 Subject: [PATCH 02/17] Assign specific jobs to dedicated workers (#564) * separate workers * fix review comments * add worker run modes * change default queue * update cargo.lock * fix review comments * testing changes * separate segment proof and segment agg proof * review comments * fix review comments * Update zero/src/bin/leader/cli.rs Co-authored-by: Robin Salen <30937548+Nashtare@users.noreply.github.com> * update paladin version --------- Co-authored-by: Ben Co-authored-by: Robin Salen <30937548+Nashtare@users.noreply.github.com> --- Cargo.lock | 97 ++++++++++++++++------------------- Cargo.toml | 2 +- zero/src/bin/leader.rs | 41 +++++++++++++-- zero/src/bin/leader/cli.rs | 22 +++++++- zero/src/bin/leader/client.rs | 12 +++-- zero/src/bin/leader/http.rs | 16 +++--- zero/src/bin/leader/stdio.rs | 17 ++++-- zero/src/prover.rs | 46 +++++++++++------ 8 files changed, 157 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cd9634ff..d642c2107 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,9 +58,9 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c37d89f69cb43901949ba29307ada8b9e3b170f94057ad4c04d6fd169d24d65f" +checksum = "b2683873c2744f6cd72d0db51bb74fee9ed310e0476a140bdc19e82b407d8a0a" dependencies = [ "alloy-consensus", "alloy-core", @@ -76,9 +76,9 @@ dependencies = [ [[package]] name = "alloy-chains" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b4f201b0ac8f81315fbdc55269965a8ddadbc04ab47fa65a1a468f9a40f7a5f" +checksum = "b68b94c159bcc2ca5f758b8663d7b00fc7c5e40569984595ddf2221b0f7f7f6e" dependencies = [ "num_enum", "strum", @@ -96,9 +96,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1468e3128e07c7afe4ff13c17e8170c330d12c322f8924b8bf6986a27e0aad3d" +checksum = "c28ddd17ffb7e4d66ef3a84e7b179072a9320cdc4b26c7f6f44cbf1081631b36" dependencies = [ "alloy-eips", "alloy-primitives", @@ -142,9 +142,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c35df7b972b06f1b2f4e8b7a53328522fa788054a9d3e556faf2411c5a51d5a" +checksum = "2f6c5c0a383f14519531cf58d8440e74f10b938e289f803af870be6f79223110" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -160,9 +160,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8866562186d237f1dfeaf989ef941a24764f764bf5c33311e37ead3519c6a429" +checksum = "7111af869909275cffc5c84d16b6c892d6d512773e40cbe83187d0b9c5235e91" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -174,9 +174,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe714e233f9eaf410de95a9af6bcd05d3a7f8c8de7a0817221e95a6b642a080" +checksum = "342028392a2d5050b7b93dd32a0715d3b3b9ce30072ecb69a35dd4895c005495" dependencies = [ "alloy-consensus", "alloy-eips", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c5a38117974c5776a45e140226745a0b664f79736aa900995d8e4121558e064" +checksum = "a6e66d78c049dcadd065a926a9f2d9a9b2b10981a7889449e694fac7bccd2c6f" dependencies = [ "alloy-eips", "alloy-primitives", @@ -229,9 +229,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c65633d6ef83c3626913c004eaf166a6dd50406f724772ea8567135efd6dc5d3" +checksum = "79f14ccc2a3c575cb17b1b4af8c772cf9b5b93b7ce7047d6640e53954abb558d" dependencies = [ "alloy-chains", "alloy-consensus", @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fc328bb5d440599ba1b5aa44c0b9ab0625fbc3a403bb5ee94ed4a01ba23e07" +checksum = "4dc79aeca84abb122a2fffbc1c91fdf958dca5c95be3875977bc99672bde0027" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -307,9 +307,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f8ff679f94c497a8383f2cd09e2a099266e5f3d5e574bc82b4b379865707dbb" +checksum = "22045187a5ebf5b2af3f8b6831b66735b6556c5750ec5790aeeb45935260c1c2" dependencies = [ "alloy-rpc-types-eth", "alloy-rpc-types-trace", @@ -319,9 +319,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a59b1d7c86e0a653e7f3d29954f6de5a2878d8cfd1f010ff93be5c2c48cd3b1" +checksum = "238f494727ff861a803bd73b1274ef788a615bf8f8c4bfada4e6df42afa275d2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -330,17 +330,19 @@ dependencies = [ "alloy-rlp", "alloy-serde", "alloy-sol-types", + "cfg-if", + "derive_more", + "hashbrown", "itertools 0.13.0", "serde", "serde_json", - "thiserror", ] [[package]] name = "alloy-rpc-types-trace" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54375e5a34ec5a2cf607f9ce98c0ece30dc76ad623afeb25d3953a8d7d30f20" +checksum = "64ca08b0ccc0861055ceb83a1db009c4c8a7f52a259e7cda7ca6ca36ec2b5ce8" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -352,9 +354,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51db8a6428a2159e01b7a43ec7aac801edd0c4db1d4de06f310c288940f16fd3" +checksum = "6b95b6f024a558593dd3b8628af03f7df2ca50e4c56839293ad0a7546e471db0" dependencies = [ "alloy-primitives", "serde", @@ -363,9 +365,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebc1760c13592b7ba3fcd964abba546b8d6a9f10d15e8d92a8263731be33f36" +checksum = "da64740ff0518606c514eb0e03dd0a1daa8ff94d6d491a626fd8e50efd6c4f18" dependencies = [ "alloy-primitives", "async-trait", @@ -435,9 +437,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd5dc4e902f1860d54952446d246ac05386311ad61030a2b906ae865416d36e0" +checksum = "3c7a669caa427abe8802184c8776f5103302f9337bb30a5b36bdebc332946c14" dependencies = [ "alloy-json-rpc", "base64", @@ -454,9 +456,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1742b94bb814f1ca6b322a6f9dd38a0252ff45a3119e40e888fb7029afa500ce" +checksum = "4433ffa97aab6ae643de81c7bde9a2f043496f27368a607405a5c78a610caf74" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -572,9 +574,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.88" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e1496f8fb1fbf272686b8d37f523dab3e4a7443300055e74cdaa449f3114356" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" dependencies = [ "backtrace", ] @@ -3267,9 +3269,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "33ea5043e58958ee56f3e15a90aee535795cd7dfd319846288d93c5b57d85cbe" [[package]] name = "oorandom" @@ -3357,9 +3359,9 @@ dependencies = [ [[package]] name = "paladin-core" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5af1955eaab1506a43d046628c218b7b3915539554838feb85ed31f54aace2f2" +checksum = "6343e2767eb6ac7f55ca8b869df03c2986b88f0b9ad09cada898b3770ac436e3" dependencies = [ "anyhow", "async-trait", @@ -3377,6 +3379,7 @@ dependencies = [ "paladin-opkind-derive", "pin-project", "postcard", + "rand", "serde", "thiserror", "tokio", @@ -3385,14 +3388,13 @@ dependencies = [ "tokio-util", "tracing", "tracing-subscriber", - "uuid", ] [[package]] name = "paladin-opkind-derive" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af25dcb10b7c0ce99abee8694e2e79e4787d7f778b9339dc5a50ba6fc45e5cc9" +checksum = "0bde85d55c108d4eef3404b4c88d1982a270dc11146a085f4f45c1548b4d2b4c" dependencies = [ "quote", "syn 2.0.77", @@ -5343,17 +5345,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" -dependencies = [ - "getrandom", - "rand", - "serde", -] - [[package]] name = "valuable" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 1f952c7b4..306530217 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,7 @@ num-bigint = "0.4.5" num-traits = "0.2.19" nunny = "0.2.1" once_cell = "1.19.0" -paladin-core = "0.4.2" +paladin-core = "0.4.3" parking_lot = "0.12.3" paste = "1.0.15" pest = "2.7.10" diff --git a/zero/src/bin/leader.rs b/zero/src/bin/leader.rs index 728f6bb0e..6daaf599f 100644 --- a/zero/src/bin/leader.rs +++ b/zero/src/bin/leader.rs @@ -6,10 +6,11 @@ use anyhow::Result; use clap::Parser; use cli::Command; use client::RpcParams; +use paladin::config::Config; use paladin::runtime::Runtime; use tracing::info; use zero::env::load_dotenvy_vars_if_present; -use zero::prover::ProverConfig; +use zero::prover::{ProofRuntime, ProverConfig}; use zero::{ block_interval::BlockInterval, prover_state::persistence::set_circuit_cache_dir_env_if_not_set, }; @@ -24,6 +25,10 @@ mod leader { pub mod stdio; } +const HEAVY_PROOF_ROUTING_KEY: &str = "heavy-proof"; +const LIGHT_PROOF_ROUTING_KEY: &str = "light-proof"; +const DEFAULT_ROUTING_KEY: &str = paladin::runtime::DEFAULT_ROUTING_KEY; + #[tokio::main] async fn main() -> Result<()> { load_dotenvy_vars_if_present(); @@ -36,7 +41,33 @@ async fn main() -> Result<()> { return zero::prover_state::persistence::delete_all(); } - let runtime = Arc::new(Runtime::from_config(&args.paladin, register()).await?); + let mut light_proof_routing_key = DEFAULT_ROUTING_KEY.to_string(); + let mut heavy_proof_routing_key = DEFAULT_ROUTING_KEY.to_string(); + if args.worker_run_mode == cli::WorkerRunMode::Affinity { + // If we're running in affinity mode, we need to set the routing key for the + // heavy proof and light proof. + info!("Workers running in affinity mode"); + light_proof_routing_key = LIGHT_PROOF_ROUTING_KEY.to_string(); + heavy_proof_routing_key = HEAVY_PROOF_ROUTING_KEY.to_string(); + } + + let light_proof_paladin_args = Config { + task_bus_routing_key: Some(light_proof_routing_key), + ..args.paladin.clone() + }; + + let heavy_proof_paladin_args = Config { + task_bus_routing_key: Some(heavy_proof_routing_key), + ..args.paladin + }; + + let light_proof = Runtime::from_config(&light_proof_paladin_args, register()).await?; + let heavy_proof = Runtime::from_config(&heavy_proof_paladin_args, register()).await?; + + let proof_runtime = Arc::new(ProofRuntime { + light_proof, + heavy_proof, + }); let prover_config: ProverConfig = args.prover_config.into(); if prover_config.block_pool_size == 0 { panic!("block-pool-size must be greater than 0"); @@ -55,7 +86,7 @@ async fn main() -> Result<()> { match args.command { Command::Stdio { previous_proof } => { let previous_proof = get_previous_proof(previous_proof)?; - stdio::stdio_main(runtime, previous_proof, Arc::new(prover_config)).await?; + stdio::stdio_main(proof_runtime, previous_proof, Arc::new(prover_config)).await?; } Command::Http { port, output_dir } => { // check if output_dir exists, is a directory, and is writable @@ -67,7 +98,7 @@ async fn main() -> Result<()> { panic!("output-dir is not a writable directory"); } - http::http_main(runtime, port, output_dir, Arc::new(prover_config)).await?; + http::http_main(proof_runtime, port, output_dir, Arc::new(prover_config)).await?; } Command::Rpc { rpc_url, @@ -84,7 +115,7 @@ async fn main() -> Result<()> { info!("Proving interval {block_interval}"); client_main( - runtime, + proof_runtime, RpcParams { rpc_url, rpc_type, diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index 3569efc41..ad9270ee8 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -1,11 +1,13 @@ use std::path::PathBuf; use alloy::transports::http::reqwest::Url; -use clap::{Parser, Subcommand, ValueHint}; +use clap::{Parser, Subcommand, ValueEnum, ValueHint}; use zero::prover::cli::CliProverConfig; use zero::prover_state::cli::CliProverStateConfig; use zero::rpc::RpcType; +const WORKER_HELP_HEADING: &str = "Worker Config options"; + /// zero-bin leader config #[derive(Parser)] #[command(version = zero::version(), propagate_version = true)] @@ -23,6 +25,24 @@ pub(crate) struct Cli { // mode. #[clap(flatten)] pub(crate) prover_state_config: CliProverStateConfig, + + // Mode to use for worker for setup (affinity or default) + #[arg(long = "worker-run-mode", help_heading = WORKER_HELP_HEADING, value_enum, default_value = "default")] + pub(crate) worker_run_mode: WorkerRunMode, +} + +/// Defines the mode for worker setup in terms of job allocation: +/// +/// - `Affinity`: Workers are assigned specific types of jobs based on their +/// capabilities, distinguishing between heavy and light jobs. +/// - `Default`: No job distinction is made — any worker can handle any type of +/// job, whether heavy or light. +/// +/// This enum allows for flexible worker configuration based on workload needs. +#[derive(ValueEnum, Clone, PartialEq, Debug)] +pub enum WorkerRunMode { + Affinity, + Default, } #[derive(Subcommand)] diff --git a/zero/src/bin/leader/client.rs b/zero/src/bin/leader/client.rs index 455cceb8b..343a2cdcb 100644 --- a/zero/src/bin/leader/client.rs +++ b/zero/src/bin/leader/client.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use alloy::rpc::types::{BlockId, BlockNumberOrTag}; use alloy::transports::http::reqwest::Url; use anyhow::{anyhow, Result}; -use paladin::runtime::Runtime; use tokio::sync::mpsc; use tracing::info; use zero::block_interval::{BlockInterval, BlockIntervalStream}; @@ -13,6 +12,8 @@ use zero::prover::{self, BlockProverInput, ProverConfig}; use zero::rpc; use zero::rpc::{retry::build_http_retry_provider, RpcType}; +use crate::ProofRuntime; + #[derive(Debug)] pub struct RpcParams { pub rpc_url: Url, @@ -31,7 +32,7 @@ pub struct LeaderConfig { /// The main function for the client. pub(crate) async fn client_main( - runtime: Arc, + proof_runtime: Arc, rpc_params: RpcParams, block_interval: BlockInterval, mut leader_config: LeaderConfig, @@ -63,10 +64,10 @@ pub(crate) async fn client_main( let (block_tx, block_rx) = mpsc::channel::<(BlockProverInput, bool)>(zero::BLOCK_CHANNEL_SIZE); // Run proving task - let runtime_ = runtime.clone(); + let proof_runtime_ = proof_runtime.clone(); let proving_task = tokio::spawn(prover::prove( block_rx, - runtime_, + proof_runtime_, leader_config.previous_proof.take(), Arc::new(leader_config.prover_config), )); @@ -112,7 +113,8 @@ pub(crate) async fn client_main( } } - runtime.close().await?; + proof_runtime.light_proof.close().await?; + proof_runtime.heavy_proof.close().await?; if test_only { info!("All proof witnesses have been generated successfully."); diff --git a/zero/src/bin/leader/http.rs b/zero/src/bin/leader/http.rs index 02e968ec0..0bc90ac40 100644 --- a/zero/src/bin/leader/http.rs +++ b/zero/src/bin/leader/http.rs @@ -3,16 +3,17 @@ use std::{net::SocketAddr, path::PathBuf, sync::Arc}; use alloy::primitives::U256; use anyhow::{bail, Result}; use axum::{http::StatusCode, routing::post, Json, Router}; -use paladin::runtime::Runtime; use serde::{Deserialize, Serialize}; use serde_json::to_writer; use tracing::{debug, error, info}; use zero::proof_types::GeneratedBlockProof; use zero::prover::{BlockProverInput, ProverConfig}; +use crate::ProofRuntime; + /// The main function for the HTTP mode. pub(crate) async fn http_main( - runtime: Arc, + proof_runtime: Arc, port: u16, output_dir: PathBuf, prover_config: Arc, @@ -22,10 +23,7 @@ pub(crate) async fn http_main( let app = Router::new().route( "/prove", - post({ - let runtime = runtime.clone(); - move |body| prove(body, runtime, output_dir.clone(), prover_config) - }), + post(move |body| prove(body, proof_runtime, output_dir.clone(), prover_config)), ); let listener = tokio::net::TcpListener::bind(&addr).await?; Ok(axum::serve(listener, app).await?) @@ -62,7 +60,7 @@ struct HttpProverInput { async fn prove( Json(payload): Json, - runtime: Arc, + proof_runtime: Arc, output_dir: PathBuf, prover_config: Arc, ) -> StatusCode { @@ -74,7 +72,7 @@ async fn prove( payload .prover_input .prove_test( - runtime, + proof_runtime, payload.previous.map(futures::future::ok), prover_config, ) @@ -83,7 +81,7 @@ async fn prove( payload .prover_input .prove( - runtime, + proof_runtime, payload.previous.map(futures::future::ok), prover_config, ) diff --git a/zero/src/bin/leader/stdio.rs b/zero/src/bin/leader/stdio.rs index 306a08c65..9e451e13f 100644 --- a/zero/src/bin/leader/stdio.rs +++ b/zero/src/bin/leader/stdio.rs @@ -2,15 +2,16 @@ use std::io::Read; use std::sync::Arc; use anyhow::{anyhow, Result}; -use paladin::runtime::Runtime; use tokio::sync::mpsc; use tracing::info; use zero::proof_types::GeneratedBlockProof; use zero::prover::{self, BlockProverInput, ProverConfig}; +use crate::ProofRuntime; + /// The main function for the stdio mode. pub(crate) async fn stdio_main( - runtime: Arc, + proof_runtime: Arc, previous: Option, prover_config: Arc, ) -> Result<()> { @@ -24,9 +25,14 @@ pub(crate) async fn stdio_main( let (block_tx, block_rx) = mpsc::channel::<(BlockProverInput, bool)>(zero::BLOCK_CHANNEL_SIZE); - let runtime_ = runtime.clone(); + let proof_runtime_ = proof_runtime.clone(); let prover_config_ = prover_config.clone(); - let proving_task = tokio::spawn(prover::prove(block_rx, runtime_, previous, prover_config_)); + let proving_task = tokio::spawn(prover::prove( + block_rx, + proof_runtime_, + previous, + prover_config_, + )); let interval_len = block_prover_inputs.len(); for (index, block_prover_input) in block_prover_inputs.into_iter().enumerate() { @@ -48,7 +54,8 @@ pub(crate) async fn stdio_main( } } - runtime.close().await?; + proof_runtime.light_proof.close().await?; + proof_runtime.heavy_proof.close().await?; if prover_config.test_only { info!("All proof witnesses have been generated successfully."); diff --git a/zero/src/prover.rs b/zero/src/prover.rs index 6a194ddf1..4e221709c 100644 --- a/zero/src/prover.rs +++ b/zero/src/prover.rs @@ -9,9 +9,13 @@ use std::sync::Arc; use alloy::primitives::U256; use anyhow::{Context, Result}; use evm_arithmetization::Field; -use futures::{future::BoxFuture, FutureExt, TryFutureExt, TryStreamExt}; +use evm_arithmetization::SegmentDataIterator; +use futures::{ + future, future::BoxFuture, stream::FuturesUnordered, FutureExt, TryFutureExt, TryStreamExt, +}; use hashbrown::HashMap; use num_traits::ToPrimitive as _; +use paladin::directive::{Directive, IndexedStream}; use paladin::runtime::Runtime; use plonky2::gates::noop::NoopGate; use plonky2::plonk::circuit_builder::CircuitBuilder; @@ -28,6 +32,18 @@ use crate::fs::generate_block_proof_file_name; use crate::ops; use crate::proof_types::GeneratedBlockProof; +/// `ProofRuntime` represents the runtime environments used for generating +/// different types of proofs. It contains separate runtimes for handling: +/// +/// - `light_proof`: Typically for smaller, less resource-intensive tasks, such +/// as aggregation. +/// - `heavy_proof`: For larger, more computationally expensive tasks, such as +/// STARK proof generation. +pub struct ProofRuntime { + pub light_proof: Runtime, + pub heavy_proof: Runtime, +} + // All proving tasks are executed concurrently, which can cause issues for large // block intervals, where distant future blocks may be proven first. // @@ -65,14 +81,11 @@ impl BlockProverInput { pub async fn prove( self, - runtime: Arc, + proof_runtime: Arc, previous: Option>>, prover_config: Arc, ) -> Result { use anyhow::Context as _; - use evm_arithmetization::SegmentDataIterator; - use futures::{stream::FuturesUnordered, FutureExt}; - use paladin::directive::{Directive, IndexedStream}; let ProverConfig { max_cpu_len_log, @@ -116,7 +129,7 @@ impl BlockProverInput { Directive::map(IndexedStream::from(segment_data_iterator), &seg_prove_ops) .fold(&seg_agg_ops) - .run(&runtime) + .run(&proof_runtime.heavy_proof) .map(move |e| { e.map(|p| (idx, crate::proof_types::BatchAggregatableProof::from(p))) }) @@ -126,7 +139,7 @@ impl BlockProverInput { // Fold the batch aggregated proof stream into a single proof. let final_batch_proof = Directive::fold(IndexedStream::new(batch_proof_futs), &batch_agg_ops) - .run(&runtime) + .run(&proof_runtime.light_proof) .await?; if let crate::proof_types::BatchAggregatableProof::BatchAgg(proof) = final_batch_proof { @@ -143,7 +156,7 @@ impl BlockProverInput { prev, save_inputs_on_error, }) - .run(&runtime) + .run(&proof_runtime.light_proof) .await?; info!("Successfully proved block {block_number}"); @@ -156,13 +169,12 @@ impl BlockProverInput { pub async fn prove_test( self, - runtime: Arc, + proof_runtime: Arc, previous: Option>>, prover_config: Arc, ) -> Result { use std::iter::repeat; - use futures::future; use paladin::directive::{Directive, IndexedStream}; let ProverConfig { @@ -202,7 +214,7 @@ impl BlockProverInput { ); simulation - .run(&runtime) + .run(&proof_runtime.light_proof) .await? .try_for_each(|_| future::ok(())) .await?; @@ -236,17 +248,17 @@ impl BlockProverInput { async fn prove_block( block: BlockProverInput, - runtime: Arc, + proof_runtime: Arc, previous_block_proof: Option>>, prover_config: Arc, ) -> Result { if prover_config.test_only { block - .prove_test(runtime, previous_block_proof, prover_config) + .prove_test(proof_runtime, previous_block_proof, prover_config) .await } else { block - .prove(runtime, previous_block_proof, prover_config) + .prove(proof_runtime, previous_block_proof, prover_config) .await } } @@ -257,7 +269,7 @@ async fn prove_block( /// block proofs as well. pub async fn prove( mut block_receiver: Receiver<(BlockProverInput, bool)>, - runtime: Arc, + proof_runtime: Arc, checkpoint_proof: Option, prover_config: Arc, ) -> Result<()> { @@ -277,7 +289,7 @@ pub async fn prove( let (tx, rx) = oneshot::channel::(); let prover_config = prover_config.clone(); let previous_block_proof = prev_proof.take(); - let runtime = runtime.clone(); + let proof_runtime = proof_runtime.clone(); let block_number = block_prover_input.get_block_number(); let prove_permit = PARALLEL_BLOCK_PROVING_PERMIT_POOL.acquire().await?; @@ -288,7 +300,7 @@ pub async fn prove( // Prove the block let block_proof = prove_block( block_prover_input, - runtime, + proof_runtime, previous_block_proof, prover_config.clone(), ) From fa1e2f914d338d450f97d8c513136eaeb18d1324 Mon Sep 17 00:00:00 2001 From: 0xaatif <169152398+0xaatif@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:34:36 +0100 Subject: [PATCH 03/17] feat: SMT support in `trace_decoder` ignores storage (#693) --- .github/workflows/lint.yml | 12 +- Cargo.lock | 11 + Cargo.toml | 3 - trace_decoder/Cargo.toml | 3 +- trace_decoder/benches/block_processing.rs | 2 + trace_decoder/src/core.rs | 224 +++++++++--- trace_decoder/src/lib.rs | 8 +- trace_decoder/src/observer.rs | 2 +- trace_decoder/src/{typed_mpt.rs => tries.rs} | 345 ++++++++++++++---- trace_decoder/src/type1.rs | 15 +- trace_decoder/src/type2.rs | 294 ++++++++------- trace_decoder/src/wire.rs | 8 +- trace_decoder/tests/consistent-with-header.rs | 2 + trace_decoder/tests/simulate-execution.rs | 16 +- zero/Cargo.toml | 1 + zero/src/bin/rpc.rs | 2 + zero/src/bin/trie_diff.rs | 2 + zero/src/prover.rs | 16 +- 18 files changed, 681 insertions(+), 285 deletions(-) rename trace_decoder/src/{typed_mpt.rs => tries.rs} (58%) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 446d6a314..13089b3d0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,8 +13,6 @@ concurrency: env: CARGO_TERM_COLOR: always - BINSTALL_NO_CONFIRM: true - RUSTDOCFLAGS: "-D warnings" jobs: clippy: @@ -42,7 +40,15 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ./.github/actions/rust - - run: cargo doc --all --no-deps + - run: RUSTDOCFLAGS='-D warnings -A rustdoc::private_intra_doc_links' cargo doc --all --no-deps + # TODO(zero): https://github.com/0xPolygonZero/zk_evm/issues/718 + - run: > + RUSTDOCFLAGS='-D warnings -A rustdoc::private_intra_doc_links' cargo doc --no-deps --document-private-items + --package trace_decoder + --package compat + --package smt_trie + --package zk_evm_proc_macro + --package zk_evm_common cargo-fmt: runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/Cargo.lock b/Cargo.lock index d642c2107..ea4fb8060 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1151,6 +1151,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "build-array" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ef4e2687af237b2646687e19a0643bc369878216122e46c3f1a01c56baa9d5" +dependencies = [ + "arrayvec", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -5126,6 +5135,7 @@ dependencies = [ "assert2", "bitflags 2.6.0", "bitvec", + "build-array", "bytes", "camino", "ciborium", @@ -5800,6 +5810,7 @@ dependencies = [ "anyhow", "async-stream", "axum", + "cfg-if", "clap", "compat", "directories", diff --git a/Cargo.toml b/Cargo.toml index 306530217..e47c71386 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,6 @@ axum = "0.7.5" bitflags = "2.5.0" bitvec = "1.0.1" bytes = "1.6.0" -cargo_metadata = "0.18.1" ciborium = "0.2.2" ciborium-io = "0.2.2" clap = { version = "4.5.7", features = ["derive", "env"] } @@ -72,7 +71,6 @@ nunny = "0.2.1" once_cell = "1.19.0" paladin-core = "0.4.3" parking_lot = "0.12.3" -paste = "1.0.15" pest = "2.7.10" pest_derive = "2.7.10" pretty_env_logger = "0.5.0" @@ -94,7 +92,6 @@ syn = "2.0" thiserror = "1.0.61" tiny-keccak = "2.0.2" tokio = { version = "1.38.0", features = ["full"] } -toml = "0.8.14" tower = "0.4" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/trace_decoder/Cargo.toml b/trace_decoder/Cargo.toml index de9b389a2..5bdd24f12 100644 --- a/trace_decoder/Cargo.toml +++ b/trace_decoder/Cargo.toml @@ -15,6 +15,7 @@ alloy-compat = "0.1.0" anyhow.workspace = true bitflags.workspace = true bitvec.workspace = true +build-array = "0.1.2" bytes.workspace = true ciborium.workspace = true ciborium-io.workspace = true @@ -33,6 +34,7 @@ nunny = { workspace = true, features = ["serde"] } plonky2.workspace = true rlp.workspace = true serde.workspace = true +smt_trie.workspace = true stackstack = "0.3.0" strum = { version = "0.26.3", features = ["derive"] } thiserror.workspace = true @@ -52,7 +54,6 @@ libtest-mimic = "0.7.3" plonky2_maybe_rayon.workspace = true serde_json.workspace = true serde_path_to_error.workspace = true -smt_trie.workspace = true zero.workspace = true [features] diff --git a/trace_decoder/benches/block_processing.rs b/trace_decoder/benches/block_processing.rs index adefdae3f..4e3582e98 100644 --- a/trace_decoder/benches/block_processing.rs +++ b/trace_decoder/benches/block_processing.rs @@ -8,6 +8,7 @@ use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; use trace_decoder::observer::DummyObserver; use trace_decoder::{BlockTrace, OtherBlockData}; +use zero::prover::WIRE_DISPOSITION; #[derive(Clone, Debug, serde::Deserialize)] pub struct ProverInput { @@ -39,6 +40,7 @@ fn criterion_benchmark(c: &mut Criterion) { other_data, batch_size, &mut DummyObserver::new(), + WIRE_DISPOSITION, ) .unwrap() }, diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index c5aa890da..46495030f 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -6,6 +6,7 @@ use std::{ use alloy_compat::Compat as _; use anyhow::{anyhow, bail, ensure, Context as _}; +use either::Either; use ethereum_types::{Address, U256}; use evm_arithmetization::{ generation::{mpt::AccountRlp, TrieInputs}, @@ -18,20 +19,35 @@ use mpt_trie::partial_trie::PartialTrie as _; use nunny::NonEmpty; use zk_evm_common::gwei_to_wei; -use crate::observer::Observer; use crate::{ - typed_mpt::{ReceiptTrie, StateMpt, StateTrie, StorageTrie, TransactionTrie, TrieKey}, + observer::{DummyObserver, Observer}, + tries::StateSmt, +}; +use crate::{ + tries::{MptKey, ReceiptTrie, StateMpt, StateTrie, StorageTrie, TransactionTrie}, BlockLevelData, BlockTrace, BlockTraceTriePreImages, CombinedPreImages, ContractCodeUsage, OtherBlockData, SeparateStorageTriesPreImage, SeparateTriePreImage, SeparateTriePreImages, TxnInfo, TxnMeta, TxnTrace, }; +/// Expected trie type when parsing from binary in a [`BlockTrace`]. +/// +/// See [`crate::wire`] and [`CombinedPreImages`] for more. +#[derive(Debug)] +pub enum WireDisposition { + /// MPT + Type1, + /// SMT + Type2, +} + /// TODO(0xaatif): document this after pub fn entrypoint( trace: BlockTrace, other: OtherBlockData, batch_size_hint: usize, observer: &mut impl Observer, + wire_disposition: WireDisposition, ) -> anyhow::Result> { ensure!(batch_size_hint != 0); @@ -45,8 +61,8 @@ pub fn entrypoint( BlockTraceTriePreImages::Separate(_) => FatalMissingCode(true), BlockTraceTriePreImages::Combined(_) => FatalMissingCode(false), }; + let (state, storage, mut code) = start(trie_pre_images, wire_disposition)?; - let (state, storage, mut code) = start(trie_pre_images)?; code.extend(code_db); let OtherBlockData { @@ -66,17 +82,40 @@ pub fn entrypoint( *amt = gwei_to_wei(*amt) } - let batches = middle( - state, - storage, - batch(txn_info, batch_size_hint), - &mut code, - &b_meta, - ger_data, - withdrawals, - fatal_missing_code, - observer, - )?; + let batches = match state { + Either::Left(mpt) => Either::Left( + middle( + mpt, + storage, + batch(txn_info, batch_size_hint), + &mut code, + &b_meta, + ger_data, + withdrawals, + fatal_missing_code, + observer, + )? + .into_iter() + .map(|it| it.map(Either::Left)), + ), + Either::Right(smt) => { + Either::Right( + middle( + smt, + storage, + batch(txn_info, batch_size_hint), + &mut code, + &b_meta, + ger_data, + withdrawals, + fatal_missing_code, + &mut DummyObserver::new(), // TODO(0xaatif) + )? + .into_iter() + .map(|it| it.map(Either::Right)), + ) + } + }; let mut running_gas_used = 0; Ok(batches @@ -107,7 +146,10 @@ pub fn entrypoint( withdrawals, ger_data, tries: TrieInputs { - state_trie: state.into(), + state_trie: match state { + Either::Left(mpt) => mpt.into(), + Either::Right(_) => todo!("evm_arithmetization accepts an SMT"), + }, transactions_trie: transaction.into(), receipts_trie: receipt.into(), storage_tries: storage.into_iter().map(|(k, v)| (k, v.into())).collect(), @@ -131,11 +173,16 @@ pub fn entrypoint( /// [`HashedPartialTrie`](mpt_trie::partial_trie::HashedPartialTrie), /// or a [`wire`](crate::wire)-encoded representation of one. /// -/// Turn either of those into our [`typed_mpt`](crate::typed_mpt) -/// representations. +/// Turn either of those into our [internal representations](crate::tries). +#[allow(clippy::type_complexity)] fn start( pre_images: BlockTraceTriePreImages, -) -> anyhow::Result<(StateMpt, BTreeMap, Hash2Code)> { + wire_disposition: WireDisposition, +) -> anyhow::Result<( + Either, + BTreeMap, + Hash2Code, +)> { Ok(match pre_images { // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/401 // refactor our convoluted input types @@ -146,7 +193,7 @@ fn start( let state = state.items().try_fold( StateMpt::default(), |mut acc, (nibbles, hash_or_val)| { - let path = TrieKey::from_nibbles(nibbles); + let path = MptKey::from_nibbles(nibbles); match hash_or_val { mpt_trie::trie_ops::ValOrHash::Val(bytes) => { #[expect(deprecated)] // this is MPT specific @@ -169,7 +216,7 @@ fn start( .map(|(k, SeparateTriePreImage::Direct(v))| { v.items() .try_fold(StorageTrie::default(), |mut acc, (nibbles, hash_or_val)| { - let path = TrieKey::from_nibbles(nibbles); + let path = MptKey::from_nibbles(nibbles); match hash_or_val { mpt_trie::trie_ops::ValOrHash::Val(value) => { acc.insert(path, value)?; @@ -183,17 +230,35 @@ fn start( .map(|v| (k, v)) }) .collect::>()?; - (state, storage, Hash2Code::new()) + (Either::Left(state), storage, Hash2Code::new()) } BlockTraceTriePreImages::Combined(CombinedPreImages { compact }) => { let instructions = crate::wire::parse(&compact) .context("couldn't parse instructions from binary format")?; - let crate::type1::Frontend { - state, - storage, - code, - } = crate::type1::frontend(instructions)?; - (state, storage, code.into_iter().map(Into::into).collect()) + let (state, storage, code) = match wire_disposition { + WireDisposition::Type1 => { + let crate::type1::Frontend { + state, + storage, + code, + } = crate::type1::frontend(instructions)?; + ( + Either::Left(state), + storage, + Hash2Code::from_iter(code.into_iter().map(NonEmpty::into_vec)), + ) + } + WireDisposition::Type2 => { + let crate::type2::Frontend { trie, code } = + crate::type2::frontend(instructions)?; + ( + Either::Right(trie), + BTreeMap::new(), + Hash2Code::from_iter(code.into_iter().map(NonEmpty::into_vec)), + ) + } + }; + (state, storage, code) } }) } @@ -267,6 +332,29 @@ struct Batch { pub withdrawals: Vec<(Address, U256)>, } +impl Batch { + fn map(self, f: impl FnMut(T) -> U) -> Batch { + let Self { + first_txn_ix, + gas_used, + contract_code, + byte_code, + before, + after, + withdrawals, + } = self; + Batch { + first_txn_ix, + gas_used, + contract_code, + byte_code, + before: before.map(f), + after, + withdrawals, + } + } +} + /// [`evm_arithmetization::generation::TrieInputs`], /// generic over state trie representation. #[derive(Debug)] @@ -277,6 +365,22 @@ pub struct IntraBlockTries { pub receipt: ReceiptTrie, } +impl IntraBlockTries { + fn map(self, mut f: impl FnMut(T) -> U) -> IntraBlockTries { + let Self { + state, + storage, + transaction, + receipt, + } = self; + IntraBlockTries { + state: f(state), + storage, + transaction, + receipt, + } + } +} /// Hacky handling of possibly missing contract bytecode in `Hash2Code` inner /// map. /// Allows incomplete payloads fetched with the zero tracer to skip these @@ -303,12 +407,15 @@ fn middle( fatal_missing_code: FatalMissingCode, // called with the untrimmed tries after each batch observer: &mut impl Observer, -) -> anyhow::Result>> { +) -> anyhow::Result>> +where + StateTrieT::Key: Ord + From
, +{ // Initialise the storage tries. for (haddr, acct) in state_trie.iter() { let storage = storage_tries.entry(haddr).or_insert({ let mut it = StorageTrie::default(); - it.insert_hash(TrieKey::default(), acct.storage_root) + it.insert_hash(MptKey::default(), acct.storage_root) .expect("empty trie insert cannot fail"); it }); @@ -343,8 +450,8 @@ fn middle( // We want to perform mask the TrieInputs above, // but won't know the bounds until after the loop below, // so store that information here. - let mut storage_masks = BTreeMap::<_, BTreeSet>::new(); - let mut state_mask = BTreeSet::new(); + let mut storage_masks = BTreeMap::<_, BTreeSet>::new(); + let mut state_mask = BTreeSet::::new(); if txn_ix == 0 { do_pre_execution( @@ -440,7 +547,7 @@ fn middle( storage_written .keys() .chain(&storage_read) - .map(|it| TrieKey::from_hash(keccak_hash::keccak(it))), + .map(|it| MptKey::from_hash(keccak_hash::keccak(it))), ); if do_writes { @@ -487,7 +594,7 @@ fn middle( }; for (k, v) in storage_written { - let slot = TrieKey::from_hash(keccak_hash::keccak(k)); + let slot = MptKey::from_hash(keccak_hash::keccak(k)); match v.is_zero() { // this is actually a delete true => storage_mask.extend(storage.reporting_remove(slot)?), @@ -500,10 +607,10 @@ fn middle( } state_trie.insert_by_address(addr, acct)?; - state_mask.insert(TrieKey::from_address(addr)); + state_mask.insert(::from(addr)); } else { // Simple state access - state_mask.insert(TrieKey::from_address(addr)); + state_mask.insert(::from(addr)); } if self_destructed { @@ -526,7 +633,7 @@ fn middle( withdrawals: match loop_ix == loop_len { true => { for (addr, amt) in &withdrawals { - state_mask.insert(TrieKey::from_address(*addr)); + state_mask.insert(::from(*addr)); let mut acct = state_trie .get_by_address(*addr) .context(format!("missing address {addr:x} for withdrawal"))?; @@ -584,10 +691,13 @@ fn do_pre_execution( block: &BlockMetadata, ger_data: Option<(H256, H256)>, storage: &mut BTreeMap, - trim_storage: &mut BTreeMap>, - trim_state: &mut BTreeSet, + trim_storage: &mut BTreeMap>, + trim_state: &mut BTreeSet, state_trie: &mut StateTrieT, -) -> anyhow::Result<()> { +) -> anyhow::Result<()> +where + StateTrieT::Key: From
+ Ord, +{ // Ethereum mainnet: EIP-4788 if cfg!(feature = "eth_mainnet") { return do_beacon_hook( @@ -623,10 +733,13 @@ fn do_scalable_hook( block: &BlockMetadata, ger_data: Option<(H256, H256)>, storage: &mut BTreeMap, - trim_storage: &mut BTreeMap>, - trim_state: &mut BTreeSet, + trim_storage: &mut BTreeMap>, + trim_state: &mut BTreeSet, state_trie: &mut StateTrieT, -) -> anyhow::Result<()> { +) -> anyhow::Result<()> +where + StateTrieT::Key: From
+ Ord, +{ use evm_arithmetization::testing_utils::{ ADDRESS_SCALABLE_L2, ADDRESS_SCALABLE_L2_ADDRESS_HASHED, GLOBAL_EXIT_ROOT_ADDRESS, GLOBAL_EXIT_ROOT_ADDRESS_HASHED, GLOBAL_EXIT_ROOT_STORAGE_POS, LAST_BLOCK_STORAGE_POS, @@ -641,7 +754,7 @@ fn do_scalable_hook( .context("missing scalable contract storage trie")?; let scalable_trim = trim_storage.entry(ADDRESS_SCALABLE_L2).or_default(); - let timestamp_slot_key = TrieKey::from_slot_position(U256::from(TIMESTAMP_STORAGE_POS.1)); + let timestamp_slot_key = MptKey::from_slot_position(U256::from(TIMESTAMP_STORAGE_POS.1)); let timestamp = scalable_storage .get(×tamp_slot_key) @@ -655,7 +768,7 @@ fn do_scalable_hook( (U256::from(LAST_BLOCK_STORAGE_POS.1), block.block_number), (U256::from(TIMESTAMP_STORAGE_POS.1), timestamp), ] { - let slot = TrieKey::from_slot_position(ix); + let slot = MptKey::from_slot_position(ix); // These values are never 0. scalable_storage.insert(slot, alloy::rlp::encode(u.compat()))?; @@ -668,12 +781,12 @@ fn do_scalable_hook( let mut arr = [0; 64]; (block.block_number - 1).to_big_endian(&mut arr[0..32]); U256::from(STATE_ROOT_STORAGE_POS.1).to_big_endian(&mut arr[32..64]); - let slot = TrieKey::from_hash(keccak_hash::keccak(arr)); + let slot = MptKey::from_hash(keccak_hash::keccak(arr)); scalable_storage.insert(slot, alloy::rlp::encode(prev_block_root_hash.compat()))?; scalable_trim.insert(slot); - trim_state.insert(TrieKey::from_address(ADDRESS_SCALABLE_L2)); + trim_state.insert(::from(ADDRESS_SCALABLE_L2)); let mut scalable_acct = state_trie .get_by_address(ADDRESS_SCALABLE_L2) .context("missing scalable contract address")?; @@ -694,12 +807,12 @@ fn do_scalable_hook( let mut arr = [0; 64]; arr[0..32].copy_from_slice(&root.0); U256::from(GLOBAL_EXIT_ROOT_STORAGE_POS.1).to_big_endian(&mut arr[32..64]); - let slot = TrieKey::from_hash(keccak_hash::keccak(arr)); + let slot = MptKey::from_hash(keccak_hash::keccak(arr)); ger_storage.insert(slot, alloy::rlp::encode(l1blockhash.compat()))?; ger_trim.insert(slot); - trim_state.insert(TrieKey::from_address(GLOBAL_EXIT_ROOT_ADDRESS)); + trim_state.insert(::from(GLOBAL_EXIT_ROOT_ADDRESS)); let mut ger_acct = state_trie .get_by_address(GLOBAL_EXIT_ROOT_ADDRESS) .context("missing GER contract address")?; @@ -722,11 +835,14 @@ fn do_scalable_hook( fn do_beacon_hook( block_timestamp: U256, storage: &mut BTreeMap, - trim_storage: &mut BTreeMap>, + trim_storage: &mut BTreeMap>, parent_beacon_block_root: H256, - trim_state: &mut BTreeSet, + trim_state: &mut BTreeSet, state_trie: &mut StateTrieT, -) -> anyhow::Result<()> { +) -> anyhow::Result<()> +where + StateTrieT::Key: From
+ Ord, +{ use evm_arithmetization::testing_utils::{ BEACON_ROOTS_CONTRACT_ADDRESS, BEACON_ROOTS_CONTRACT_ADDRESS_HASHED, HISTORY_BUFFER_LENGTH, }; @@ -747,7 +863,7 @@ fn do_beacon_hook( U256::from_big_endian(parent_beacon_block_root.as_bytes()), ), ] { - let slot = TrieKey::from_slot_position(ix); + let slot = MptKey::from_slot_position(ix); beacon_trim.insert(slot); match u.is_zero() { @@ -758,7 +874,7 @@ fn do_beacon_hook( } } } - trim_state.insert(TrieKey::from_address(BEACON_ROOTS_CONTRACT_ADDRESS)); + trim_state.insert(::from(BEACON_ROOTS_CONTRACT_ADDRESS)); let mut beacon_acct = state_trie .get_by_address(BEACON_ROOTS_CONTRACT_ADDRESS) .context("missing beacon contract address")?; @@ -785,7 +901,7 @@ fn map_receipt_bytes(bytes: Vec) -> anyhow::Result> { /// If there are any txns that create contracts, then they will also /// get added here as we process the deltas. struct Hash2Code { - /// Key must always be [`hash`] of value. + /// Key must always be [`hash`](keccak_hash) of value. inner: HashMap>, } diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index 049472c40..057d11e89 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -56,16 +56,12 @@ mod interface; pub use interface::*; +mod tries; mod type1; -// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 -// add backend/prod support for type 2 -#[cfg(test)] -#[allow(dead_code)] mod type2; -mod typed_mpt; mod wire; -pub use core::entrypoint; +pub use core::{entrypoint, WireDisposition}; mod core; diff --git a/trace_decoder/src/observer.rs b/trace_decoder/src/observer.rs index 320019e55..f9811e87c 100644 --- a/trace_decoder/src/observer.rs +++ b/trace_decoder/src/observer.rs @@ -4,7 +4,7 @@ use std::marker::PhantomData; use ethereum_types::{H256, U256}; use crate::core::IntraBlockTries; -use crate::typed_mpt::{ReceiptTrie, StorageTrie, TransactionTrie}; +use crate::tries::{ReceiptTrie, StorageTrie, TransactionTrie}; /// Observer API for the trace decoder. /// Observer is used to collect various debugging and metadata info diff --git a/trace_decoder/src/typed_mpt.rs b/trace_decoder/src/tries.rs similarity index 58% rename from trace_decoder/src/typed_mpt.rs rename to trace_decoder/src/tries.rs index 8baf3cf29..91add4d98 100644 --- a/trace_decoder/src/typed_mpt.rs +++ b/trace_decoder/src/tries.rs @@ -1,10 +1,12 @@ -//! Principled MPT types used in this library. +//! Principled trie types and abstractions used in this library. use core::fmt; -use std::{collections::BTreeMap, marker::PhantomData}; +use std::{cmp, collections::BTreeMap, marker::PhantomData}; +use anyhow::ensure; +use bitvec::{array::BitArray, slice::BitSlice}; use copyvec::CopyVec; -use ethereum_types::{Address, H256, U256}; +use ethereum_types::{Address, BigEndianHash as _, H256, U256}; use evm_arithmetization::generation::mpt::AccountRlp; use mpt_trie::partial_trie::{HashedPartialTrie, Node, OnOrphanedHashNode, PartialTrie as _}; use u4::{AsNibbles, U4}; @@ -30,27 +32,26 @@ impl TypedMpt { /// Insert a node which represents an out-of-band sub-trie. /// /// See [module documentation](super) for more. - fn insert_hash(&mut self, key: TrieKey, hash: H256) -> anyhow::Result<()> { + fn insert_hash(&mut self, key: MptKey, hash: H256) -> anyhow::Result<()> { self.inner.insert(key.into_nibbles(), hash)?; Ok(()) } - /// Returns an [`Error`] if the `key` crosses into a part of the trie that - /// isn't hydrated. - fn insert(&mut self, key: TrieKey, value: T) -> anyhow::Result> + /// Returns [`Err`] if the `key` crosses into a part of the trie that + /// is hashed out. + fn insert(&mut self, key: MptKey, value: T) -> anyhow::Result<()> where T: rlp::Encodable + rlp::Decodable, { - let prev = self.get(key); self.inner .insert(key.into_nibbles(), rlp::encode(&value).to_vec())?; - Ok(prev) + Ok(()) } /// Note that this returns [`None`] if `key` crosses into a part of the - /// trie that isn't hydrated. + /// trie that is hashed out. /// /// # Panics /// - If [`rlp::decode`]-ing for `T` doesn't round-trip. - fn get(&self, key: TrieKey) -> Option + fn get(&self, key: MptKey) -> Option where T: rlp::Decodable, { @@ -67,12 +68,12 @@ impl TypedMpt { self.inner.hash() } /// Note that this returns owned paths and items. - fn iter(&self) -> impl Iterator + '_ + fn iter(&self) -> impl Iterator + '_ where T: rlp::Decodable, { self.inner.keys().filter_map(|nib| { - let path = TrieKey::from_nibbles(nib); + let path = MptKey::from_nibbles(nib); Some((path, self.get(path)?)) }) } @@ -88,7 +89,7 @@ impl<'a, T> IntoIterator for &'a TypedMpt where T: rlp::Decodable, { - type Item = (TrieKey, T); + type Item = (MptKey, T); type IntoIter = Box + 'a>; fn into_iter(self) -> Self::IntoIter { Box::new(self.iter()) @@ -100,9 +101,9 @@ where /// /// Semantically equivalent to [`mpt_trie::nibbles::Nibbles`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] -pub struct TrieKey(CopyVec); +pub struct MptKey(CopyVec); -impl fmt::Display for TrieKey { +impl fmt::Display for MptKey { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { for u in self.0 { f.write_fmt(format_args!("{:x}", u))? @@ -111,9 +112,9 @@ impl fmt::Display for TrieKey { } } -impl TrieKey { +impl MptKey { pub fn new(components: impl IntoIterator) -> anyhow::Result { - Ok(TrieKey(CopyVec::try_from_iter(components)?)) + Ok(MptKey(CopyVec::try_from_iter(components)?)) } pub fn into_hash_left_padded(mut self) -> H256 { for _ in 0..self.0.spare_capacity_mut().len() { @@ -136,7 +137,7 @@ impl TrieKey { } pub fn from_txn_ix(txn_ix: usize) -> Self { - TrieKey::new(AsNibbles(rlp::encode(&txn_ix))).expect( + MptKey::new(AsNibbles(rlp::encode(&txn_ix))).expect( "\ rlp of an usize goes through a u64, which is 8 bytes, which will be 9 bytes RLP'ed. @@ -170,17 +171,111 @@ impl TrieKey { } } +impl From
for MptKey { + fn from(value: Address) -> Self { + Self::from_hash(keccak_hash::keccak(value)) + } +} + #[test] -fn key_into_hash() { - assert_eq!(TrieKey::new([]).unwrap().into_hash(), None); +fn mpt_key_into_hash() { + assert_eq!(MptKey::new([]).unwrap().into_hash(), None); assert_eq!( - TrieKey::new(itertools::repeat_n(u4::u4!(0), 64)) + MptKey::new(itertools::repeat_n(u4::u4!(0), 64)) .unwrap() .into_hash(), Some(H256::zero()) ) } +/// Bounded sequence of bits, +/// used as a key for [`StateSmt`]. +/// +/// Semantically equivalent to [`smt_trie::bits::Bits`]. +#[derive(Clone, Copy)] +pub struct SmtKey { + bits: bitvec::array::BitArray<[u8; 32]>, + len: usize, +} + +impl SmtKey { + fn as_bitslice(&self) -> &BitSlice { + self.bits.as_bitslice().get(..self.len).unwrap() + } +} + +impl fmt::Debug for SmtKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list() + .entries(self.as_bitslice().iter().map(|it| match *it { + true => 1, + false => 0, + })) + .finish() + } +} + +impl fmt::Display for SmtKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for bit in self.as_bitslice() { + f.write_str(match *bit { + true => "1", + false => "0", + })? + } + Ok(()) + } +} + +impl SmtKey { + pub fn new(components: impl IntoIterator) -> anyhow::Result { + let mut bits = bitvec::array::BitArray::default(); + let mut len = 0; + for (ix, bit) in components.into_iter().enumerate() { + ensure!( + bits.get(ix).is_some(), + "expected at most {} components", + bits.len() + ); + bits.set(ix, bit); + len += 1 + } + Ok(Self { bits, len }) + } + + pub fn into_smt_bits(self) -> smt_trie::bits::Bits { + let mut bits = smt_trie::bits::Bits::default(); + for bit in self.as_bitslice() { + bits.push_bit(*bit) + } + bits + } +} + +impl From
for SmtKey { + fn from(addr: Address) -> Self { + let H256(bytes) = keccak_hash::keccak(addr); + Self::new(BitArray::<_>::new(bytes)).expect("SmtKey has room for 256 bits") + } +} + +impl Ord for SmtKey { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.as_bitslice().cmp(other.as_bitslice()) + } +} +impl PartialOrd for SmtKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Eq for SmtKey {} +impl PartialEq for SmtKey { + fn eq(&self, other: &Self) -> bool { + self.as_bitslice().eq(other.as_bitslice()) + } +} + /// Per-block, `txn_ix -> [u8]`. /// /// See @@ -196,10 +291,10 @@ impl TransactionTrie { pub fn insert(&mut self, txn_ix: usize, val: Vec) -> anyhow::Result>> { let prev = self .untyped - .get(TrieKey::from_txn_ix(txn_ix).into_nibbles()) + .get(MptKey::from_txn_ix(txn_ix).into_nibbles()) .map(Vec::from); self.untyped - .insert(TrieKey::from_txn_ix(txn_ix).into_nibbles(), val)?; + .insert(MptKey::from_txn_ix(txn_ix).into_nibbles(), val)?; Ok(prev) } pub fn root(&self) -> H256 { @@ -214,7 +309,7 @@ impl TransactionTrie { &self.untyped, txn_ixs .into_iter() - .map(|it| TrieKey::from_txn_ix(it).into_nibbles()), + .map(|it| MptKey::from_txn_ix(it).into_nibbles()), )?; Ok(()) } @@ -241,10 +336,10 @@ impl ReceiptTrie { pub fn insert(&mut self, txn_ix: usize, val: Vec) -> anyhow::Result>> { let prev = self .untyped - .get(TrieKey::from_txn_ix(txn_ix).into_nibbles()) + .get(MptKey::from_txn_ix(txn_ix).into_nibbles()) .map(Vec::from); self.untyped - .insert(TrieKey::from_txn_ix(txn_ix).into_nibbles(), val)?; + .insert(MptKey::from_txn_ix(txn_ix).into_nibbles(), val)?; Ok(prev) } pub fn root(&self) -> H256 { @@ -259,7 +354,7 @@ impl ReceiptTrie { &self.untyped, txn_ixs .into_iter() - .map(|it| TrieKey::from_txn_ix(it).into_nibbles()), + .map(|it| MptKey::from_txn_ix(it).into_nibbles()), )?; Ok(()) } @@ -271,18 +366,14 @@ impl From for HashedPartialTrie { } } -/// TODO(0xaatif): document this after refactoring is done https://github.com/0xPolygonZero/zk_evm/issues/275 +/// TODO(0xaatif): document this after refactoring is done pub trait StateTrie { - fn insert_by_address( - &mut self, - address: Address, - account: AccountRlp, - ) -> anyhow::Result>; - fn insert_hash_by_key(&mut self, key: TrieKey, hash: H256) -> anyhow::Result<()>; + type Key; + fn insert_by_address(&mut self, address: Address, account: AccountRlp) -> anyhow::Result<()>; fn get_by_address(&self, address: Address) -> Option; - fn reporting_remove(&mut self, address: Address) -> anyhow::Result>; - /// _Hash out_ parts of the trie that aren't in `txn_ixs`. - fn mask(&mut self, address: impl IntoIterator) -> anyhow::Result<()>; + fn reporting_remove(&mut self, address: Address) -> anyhow::Result>; + /// _Hash out_ parts of the trie that aren't in `addresses`. + fn mask(&mut self, address: impl IntoIterator) -> anyhow::Result<()>; fn iter(&self) -> impl Iterator + '_; fn root(&self) -> H256; } @@ -304,13 +395,17 @@ impl StateMpt { }, } } + /// Insert a _hashed out_ part of the trie + pub fn insert_hash_by_key(&mut self, key: MptKey, hash: H256) -> anyhow::Result<()> { + self.typed.insert_hash(key, hash) + } #[deprecated = "prefer operations on `Address` where possible, as SMT support requires this"] pub fn insert_by_hashed_address( &mut self, key: H256, account: AccountRlp, - ) -> anyhow::Result> { - self.typed.insert(TrieKey::from_hash(key), account) + ) -> anyhow::Result<()> { + self.typed.insert(MptKey::from_hash(key), account) } pub fn iter(&self) -> impl Iterator + '_ { self.typed @@ -323,34 +418,27 @@ impl StateMpt { } impl StateTrie for StateMpt { - fn insert_by_address( - &mut self, - address: Address, - account: AccountRlp, - ) -> anyhow::Result> { + type Key = MptKey; + fn insert_by_address(&mut self, address: Address, account: AccountRlp) -> anyhow::Result<()> { #[expect(deprecated)] self.insert_by_hashed_address(keccak_hash::keccak(address), account) } - /// Insert an _hashed out_ part of the trie - fn insert_hash_by_key(&mut self, key: TrieKey, hash: H256) -> anyhow::Result<()> { - self.typed.insert_hash(key, hash) - } fn get_by_address(&self, address: Address) -> Option { self.typed - .get(TrieKey::from_hash(keccak_hash::keccak(address))) + .get(MptKey::from_hash(keccak_hash::keccak(address))) } /// Delete the account at `address`, returning any remaining branch on /// collapse - fn reporting_remove(&mut self, address: Address) -> anyhow::Result> { + fn reporting_remove(&mut self, address: Address) -> anyhow::Result> { delete_node_and_report_remaining_key_if_branch_collapsed( self.typed.as_mut_hashed_partial_trie_unchecked(), - TrieKey::from_address(address), + MptKey::from_address(address), ) } - fn mask(&mut self, addresses: impl IntoIterator) -> anyhow::Result<()> { + fn mask(&mut self, addresses: impl IntoIterator) -> anyhow::Result<()> { let inner = mpt_trie::trie_subsets::create_trie_subset( self.typed.as_hashed_partial_trie(), - addresses.into_iter().map(TrieKey::into_nibbles), + addresses.into_iter().map(MptKey::into_nibbles), )?; self.typed = TypedMpt { inner, @@ -377,31 +465,30 @@ impl From for HashedPartialTrie { } } +// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/706 +// We're covering for [`smt_trie`] in a couple of ways: +// - insertion operations aren't fallible, they just panic. +// - it documents a requirement that `set_hash` is called before `set`. +#[derive(Clone, Debug)] pub struct StateSmt { address2state: BTreeMap, - hashed_out: BTreeMap, + hashed_out: BTreeMap, } impl StateTrie for StateSmt { - fn insert_by_address( - &mut self, - address: Address, - account: AccountRlp, - ) -> anyhow::Result> { - Ok(self.address2state.insert(address, account)) - } - fn insert_hash_by_key(&mut self, key: TrieKey, hash: H256) -> anyhow::Result<()> { - self.hashed_out.insert(key, hash); + type Key = SmtKey; + fn insert_by_address(&mut self, address: Address, account: AccountRlp) -> anyhow::Result<()> { + self.address2state.insert(address, account); Ok(()) } fn get_by_address(&self, address: Address) -> Option { self.address2state.get(&address).copied() } - fn reporting_remove(&mut self, address: Address) -> anyhow::Result> { + fn reporting_remove(&mut self, address: Address) -> anyhow::Result> { self.address2state.remove(&address); Ok(None) } - fn mask(&mut self, address: impl IntoIterator) -> anyhow::Result<()> { + fn mask(&mut self, address: impl IntoIterator) -> anyhow::Result<()> { let _ = address; Ok(()) } @@ -411,7 +498,111 @@ impl StateTrie for StateSmt { .map(|(addr, acct)| (keccak_hash::keccak(addr), *acct)) } fn root(&self) -> H256 { - todo!() + conv_hash::smt2eth(self.as_smt().root) + } +} + +impl StateSmt { + pub(crate) fn new_unchecked( + address2state: BTreeMap, + hashed_out: BTreeMap, + ) -> Self { + Self { + address2state, + hashed_out, + } + } + + fn as_smt(&self) -> smt_trie::smt::Smt { + let Self { + address2state, + hashed_out, + } = self; + let mut smt = smt_trie::smt::Smt::::default(); + for (k, v) in hashed_out { + smt.set_hash(k.into_smt_bits(), conv_hash::eth2smt(*v)); + } + for ( + addr, + AccountRlp { + nonce, + balance, + storage_root, + code_hash, + }, + ) in address2state + { + smt.set(smt_trie::keys::key_nonce(*addr), *nonce); + smt.set(smt_trie::keys::key_balance(*addr), *balance); + smt.set(smt_trie::keys::key_code(*addr), code_hash.into_uint()); + smt.set( + // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/707 + // combined abstraction for state and storage + smt_trie::keys::key_storage(*addr, U256::zero()), + storage_root.into_uint(), + ); + } + smt + } +} + +mod conv_hash { + //! We [`u64::to_le_bytes`] because: + //! - Reference go code just puns the bytes: + //! - It's better to fix the endianness for correctness. + //! - Most (consumer) CPUs are little-endian. + + use std::array; + + use ethereum_types::H256; + use itertools::Itertools as _; + use plonky2::{ + field::{ + goldilocks_field::GoldilocksField, + types::{Field as _, PrimeField64}, + }, + hash::hash_types::HashOut, + }; + + /// # Panics + /// - On certain inputs if `debug_assertions` are enabled. See + /// [`GoldilocksField::from_canonical_u64`] for more. + pub fn eth2smt(H256(bytes): H256) -> smt_trie::smt::HashOut { + let mut bytes = bytes.into_iter(); + // (no unsafe, no unstable) + let ret = HashOut { + elements: array::from_fn(|_ix| { + let (a, b, c, d, e, f, g, h) = bytes.next_tuple().unwrap(); + GoldilocksField::from_canonical_u64(u64::from_le_bytes([a, b, c, d, e, f, g, h])) + }), + }; + assert_eq!(bytes.len(), 0); + ret + } + pub fn smt2eth(HashOut { elements }: smt_trie::smt::HashOut) -> H256 { + H256( + build_array::ArrayBuilder::from_iter( + elements + .iter() + .map(GoldilocksField::to_canonical_u64) + .flat_map(u64::to_le_bytes), + ) + .build_exact() + .unwrap(), + ) + } + + #[test] + fn test() { + use plonky2::field::types::Field64 as _; + let mut max = std::iter::repeat(GoldilocksField::ORDER - 1).flat_map(u64::to_le_bytes); + for h in [ + H256::zero(), + H256(array::from_fn(|ix| ix as u8)), + H256(array::from_fn(|_| max.next().unwrap())), + ] { + assert_eq!(smt2eth(eth2smt(h)), h); + } } } @@ -428,15 +619,15 @@ impl StorageTrie { untyped: HashedPartialTrie::new_with_strategy(Node::Empty, strategy), } } - pub fn get(&mut self, key: &TrieKey) -> Option<&[u8]> { + pub fn get(&mut self, key: &MptKey) -> Option<&[u8]> { self.untyped.get(key.into_nibbles()) } - pub fn insert(&mut self, key: TrieKey, value: Vec) -> anyhow::Result>> { + pub fn insert(&mut self, key: MptKey, value: Vec) -> anyhow::Result>> { let prev = self.get(&key).map(Vec::from); self.untyped.insert(key.into_nibbles(), value)?; Ok(prev) } - pub fn insert_hash(&mut self, key: TrieKey, hash: H256) -> anyhow::Result<()> { + pub fn insert_hash(&mut self, key: MptKey, hash: H256) -> anyhow::Result<()> { self.untyped.insert(key.into_nibbles(), hash)?; Ok(()) } @@ -446,17 +637,17 @@ impl StorageTrie { pub const fn as_hashed_partial_trie(&self) -> &HashedPartialTrie { &self.untyped } - pub fn reporting_remove(&mut self, key: TrieKey) -> anyhow::Result> { + pub fn reporting_remove(&mut self, key: MptKey) -> anyhow::Result> { delete_node_and_report_remaining_key_if_branch_collapsed(&mut self.untyped, key) } pub fn as_mut_hashed_partial_trie_unchecked(&mut self) -> &mut HashedPartialTrie { &mut self.untyped } /// _Hash out_ the parts of the trie that aren't in `paths`. - pub fn mask(&mut self, paths: impl IntoIterator) -> anyhow::Result<()> { + pub fn mask(&mut self, paths: impl IntoIterator) -> anyhow::Result<()> { self.untyped = mpt_trie::trie_subsets::create_trie_subset( &self.untyped, - paths.into_iter().map(TrieKey::into_nibbles), + paths.into_iter().map(MptKey::into_nibbles), )?; Ok(()) } @@ -473,18 +664,18 @@ impl From for HashedPartialTrie { /// plonky2. Returns the key to the remaining child if a collapse occurred. fn delete_node_and_report_remaining_key_if_branch_collapsed( trie: &mut HashedPartialTrie, - key: TrieKey, -) -> anyhow::Result> { + key: MptKey, +) -> anyhow::Result> { let old_trace = get_trie_trace(trie, key); trie.delete(key.into_nibbles())?; let new_trace = get_trie_trace(trie, key); Ok( node_deletion_resulted_in_a_branch_collapse(&old_trace, &new_trace) - .map(TrieKey::from_nibbles), + .map(MptKey::from_nibbles), ) } -fn get_trie_trace(trie: &HashedPartialTrie, k: TrieKey) -> mpt_trie::utils::TriePath { +fn get_trie_trace(trie: &HashedPartialTrie, k: MptKey) -> mpt_trie::utils::TriePath { mpt_trie::special_query::path_for_query(trie, k.into_nibbles(), true).collect() } diff --git a/trace_decoder/src/type1.rs b/trace_decoder/src/type1.rs index aeea0dbb6..c44beaec7 100644 --- a/trace_decoder/src/type1.rs +++ b/trace_decoder/src/type1.rs @@ -12,7 +12,7 @@ use mpt_trie::partial_trie::OnOrphanedHashNode; use nunny::NonEmpty; use u4::U4; -use crate::typed_mpt::{StateMpt, StateTrie as _, StorageTrie, TrieKey}; +use crate::tries::{MptKey, StateMpt, StorageTrie}; use crate::wire::{Instruction, SmtLeaf}; #[derive(Debug, Clone)] @@ -66,10 +66,10 @@ fn visit( Node::Hash(Hash { raw_hash }) => { frontend .state - .insert_hash_by_key(TrieKey::new(path.iter().copied())?, raw_hash.into())?; + .insert_hash_by_key(MptKey::new(path.iter().copied())?, raw_hash.into())?; } Node::Leaf(Leaf { key, value }) => { - let path = TrieKey::new(path.iter().copied().chain(key))? + let path = MptKey::new(path.iter().copied().chain(key))? .into_hash() .context("invalid depth for leaf of state trie")?; match value { @@ -106,8 +106,7 @@ fn visit( }, }; #[expect(deprecated)] // this is MPT-specific code - let clobbered = frontend.state.insert_by_hashed_address(path, account)?; - ensure!(clobbered.is_none(), "duplicate account"); + frontend.state.insert_by_hashed_address(path, account)?; } } } @@ -141,12 +140,12 @@ fn node2storagetrie(node: Node) -> anyhow::Result { ) -> anyhow::Result<()> { match node { Node::Hash(Hash { raw_hash }) => { - mpt.insert_hash(TrieKey::new(path.iter().copied())?, raw_hash.into())?; + mpt.insert_hash(MptKey::new(path.iter().copied())?, raw_hash.into())?; } Node::Leaf(Leaf { key, value }) => { match value { Either::Left(Value { raw_value }) => mpt.insert( - TrieKey::new(path.iter().copied().chain(key))?, + MptKey::new(path.iter().copied().chain(key))?, rlp::encode(&raw_value.as_slice()).to_vec(), )?, Either::Right(_) => bail!("unexpected account node in storage trie"), @@ -380,6 +379,8 @@ fn finish_stack(v: &mut Vec) -> anyhow::Result { #[test] fn test_tries() { + use crate::tries::StateTrie as _; + for (ix, case) in serde_json::from_str::>(include_str!("cases/zero_jerigon.json")) .unwrap() diff --git a/trace_decoder/src/type2.rs b/trace_decoder/src/type2.rs index dd3e45c4b..a71761533 100644 --- a/trace_decoder/src/type2.rs +++ b/trace_decoder/src/type2.rs @@ -1,35 +1,34 @@ //! Frontend for the witness format emitted by e.g [`0xPolygonHermez/cdk-erigon`](https://github.com/0xPolygonHermez/cdk-erigon/) //! Ethereum node. -use std::{ - collections::{HashMap, HashSet}, - iter, -}; +use std::collections::{BTreeMap, HashSet}; use anyhow::{bail, ensure, Context as _}; -use bitvec::vec::BitVec; -use either::Either; -use ethereum_types::BigEndianHash as _; -use itertools::{EitherOrBoth, Itertools as _}; +use ethereum_types::{Address, U256}; +use evm_arithmetization::generation::mpt::AccountRlp; +use itertools::EitherOrBoth; +use keccak_hash::H256; use nunny::NonEmpty; -use plonky2::field::types::Field; - -use crate::wire::{Instruction, SmtLeaf, SmtLeafType}; +use stackstack::Stack; -type SmtTrie = smt_trie::smt::Smt; +use crate::{ + tries::{SmtKey, StateSmt}, + wire::{Instruction, SmtLeaf, SmtLeafType}, +}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +/// Combination of all the [`SmtLeaf::node_type`]s +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] pub struct CollatedLeaf { pub balance: Option, pub nonce: Option, - pub code_hash: Option, - pub storage_root: Option, + pub code: Option, + pub code_length: Option, + pub storage: BTreeMap, } pub struct Frontend { - pub trie: SmtTrie, + pub trie: StateSmt, pub code: HashSet>>, - pub collation: HashMap, } /// # Panics @@ -37,18 +36,13 @@ pub struct Frontend { /// NOT call this function on untrusted inputs. pub fn frontend(instructions: impl IntoIterator) -> anyhow::Result { let (node, code) = fold(instructions).context("couldn't fold smt from instructions")?; - let (trie, collation) = - node2trie(node).context("couldn't construct trie and collation from folded node")?; - Ok(Frontend { - trie, - code, - collation, - }) + let trie = node2trie(node).context("couldn't construct trie and collation from folded node")?; + Ok(Frontend { trie, code }) } /// Node in a binary (SMT) tree. /// -/// This is an intermediary type on the way to [`SmtTrie`]. +/// This is an intermediary type on the way to [`StateSmt`]. enum Node { Branch(EitherOrBoth>), Hash([u8; 32]), @@ -105,9 +99,9 @@ fn fold1(instructions: impl IntoIterator) -> anyhow::Result< Ok(Some(match mask { // note that the single-child bits are reversed... - 0b0001 => Node::Branch(EitherOrBoth::Left(get_child()?)), - 0b0010 => Node::Branch(EitherOrBoth::Right(get_child()?)), - 0b0011 => Node::Branch(EitherOrBoth::Both(get_child()?, get_child()?)), + 0b_01 => Node::Branch(EitherOrBoth::Left(get_child()?)), + 0b_10 => Node::Branch(EitherOrBoth::Right(get_child()?)), + 0b_11 => Node::Branch(EitherOrBoth::Both(get_child()?, get_child()?)), other => bail!("unexpected bit pattern in Branch mask: {:#b}", other), })) } @@ -119,113 +113,162 @@ fn fold1(instructions: impl IntoIterator) -> anyhow::Result< } } -/// Pack a [`Node`] tree into an [`SmtTrie`]. -/// Also summarizes the [`Node::Leaf`]s out-of-band. -/// -/// # Panics -/// - if the tree is too deep. -/// - if [`SmtLeaf::address`] or [`SmtLeaf::value`] are the wrong length. -/// - if [`SmtLeafType::Storage`] is the wrong length. -/// - [`SmtTrie`] panics internally. -fn node2trie( - node: Node, -) -> anyhow::Result<(SmtTrie, HashMap)> { - let mut trie = SmtTrie::default(); - - let (hashes, leaves) = - iter_leaves(node).partition_map::, Vec<_>, _, _, _>(|(path, leaf)| match leaf { - Either::Left(it) => Either::Left((path, it)), - Either::Right(it) => Either::Right(it), - }); - - for (path, hash) in hashes { - // needs to be called before `set`, below, "to avoid any issues" according - // to the smt docs. - trie.set_hash( - bits2bits(path), - smt_trie::smt::HashOut { - elements: { - let ethereum_types::U256(arr) = ethereum_types::H256(hash).into_uint(); - arr.map(smt_trie::smt::F::from_canonical_u64) +fn node2trie(node: Node) -> anyhow::Result { + let mut hashes = BTreeMap::new(); + let mut leaves = BTreeMap::new(); + visit(&mut hashes, &mut leaves, Stack::new(), node)?; + Ok(StateSmt::new_unchecked( + leaves + .into_iter() + .map( + |( + addr, + CollatedLeaf { + balance, + nonce, + // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/707 + // we shouldn't ignore these fields + code: _, + code_length: _, + storage: _, + }, + )| { + ( + addr, + AccountRlp { + nonce: nonce.unwrap_or_default(), + balance: balance.unwrap_or_default(), + storage_root: H256::zero(), + code_hash: H256::zero(), + }, + ) }, - }, - ) - } + ) + .collect(), + hashes, + )) +} - let mut collated = HashMap::::new(); - for SmtLeaf { - node_type, - address, - value, - } in leaves - { - let address = ethereum_types::Address::from_slice(&address); - let collated = collated.entry(address).or_default(); - let value = ethereum_types::U256::from_big_endian(&value); - let key = match node_type { - SmtLeafType::Balance => { - ensure!(collated.balance.is_none(), "double write of field"); - collated.balance = Some(value); - smt_trie::keys::key_balance(address) - } - SmtLeafType::Nonce => { - ensure!(collated.nonce.is_none(), "double write of field"); - collated.nonce = Some(value); - smt_trie::keys::key_nonce(address) +fn visit( + hashes: &mut BTreeMap, + leaves: &mut BTreeMap, + path: Stack, + node: Node, +) -> anyhow::Result<()> { + match node { + Node::Branch(children) => { + let (left, right) = children.left_and_right(); + if let Some(left) = left { + visit(hashes, leaves, path.pushed(false), *left)?; } - SmtLeafType::Code => { - ensure!(collated.code_hash.is_none(), "double write of field"); - collated.code_hash = Some({ - let mut it = ethereum_types::H256::zero(); - value.to_big_endian(it.as_bytes_mut()); - it - }); - smt_trie::keys::key_code(address) + if let Some(right) = right { + visit(hashes, leaves, path.pushed(true), *right)?; } - SmtLeafType::Storage(it) => { - ensure!(collated.storage_root.is_none(), "double write of field"); - // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 - // do we not do anything with the storage here? - smt_trie::keys::key_storage(address, ethereum_types::U256::from_big_endian(&it)) + } + Node::Hash(hash) => { + hashes.insert(SmtKey::new(path.iter().copied())?, H256(hash)); + } + Node::Leaf(SmtLeaf { + node_type, + address, + value, + }) => { + let address = Address::from_slice(&address); + let collated = leaves.entry(address).or_default(); + let value = U256::from_big_endian(&value); + macro_rules! ensure { + ($expr:expr) => { + ::anyhow::ensure!($expr, "double write of field for address {}", address) + }; } - SmtLeafType::CodeLength => smt_trie::keys::key_code_length(address), - }; - trie.set(key, value) + match node_type { + SmtLeafType::Balance => { + ensure!(collated.balance.is_none()); + collated.balance = Some(value) + } + SmtLeafType::Nonce => { + ensure!(collated.nonce.is_none()); + collated.nonce = Some(value) + } + SmtLeafType::Code => { + ensure!(collated.code.is_none()); + collated.code = Some(value) + } + SmtLeafType::Storage(slot) => { + let clobbered = collated.storage.insert(U256::from_big_endian(&slot), value); + ensure!(clobbered.is_none()) + } + SmtLeafType::CodeLength => { + ensure!(collated.code_length.is_none()); + collated.code_length = Some(value) + } + }; + } } - Ok((trie, collated)) + Ok(()) } -/// # Panics -/// - on overcapacity -fn bits2bits(ours: BitVec) -> smt_trie::bits::Bits { - let mut theirs = smt_trie::bits::Bits::empty(); - for it in ours { - theirs.push_bit(it) - } - theirs -} +#[test] +fn test_tries() { + type Smt = smt_trie::smt::Smt; + use ethereum_types::BigEndianHash as _; + use plonky2::field::types::{Field, Field64 as _}; -/// Simple, inefficient visitor of all leaves of the [`Node`] tree. -#[allow(clippy::type_complexity)] -fn iter_leaves(node: Node) -> Box)>> { - match node { - Node::Hash(it) => Box::new(iter::once((BitVec::new(), Either::Left(it)))), - Node::Branch(it) => { - let (left, right) = it.left_and_right(); - let left = left - .into_iter() - .flat_map(|it| iter_leaves(*it).update(|(path, _)| path.insert(0, false))); - let right = right - .into_iter() - .flat_map(|it| iter_leaves(*it).update(|(path, _)| path.insert(0, true))); - Box::new(left.chain(right)) + // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/707 + // this logic should live in StateSmt, but we need to + // - abstract over state and storage tries + // - parameterize the account types + // we preserve this code as a tested record of how it _should_ + // be done. + fn node2trie(node: Node) -> anyhow::Result { + let mut trie = Smt::default(); + let mut hashes = BTreeMap::new(); + let mut leaves = BTreeMap::new(); + visit(&mut hashes, &mut leaves, Stack::new(), node)?; + for (key, hash) in hashes { + trie.set_hash( + key.into_smt_bits(), + smt_trie::smt::HashOut { + elements: { + let ethereum_types::U256(arr) = hash.into_uint(); + for u in arr { + ensure!(u < smt_trie::smt::F::ORDER); + } + arr.map(smt_trie::smt::F::from_canonical_u64) + }, + }, + ); } - Node::Leaf(it) => Box::new(iter::once((BitVec::new(), Either::Right(it)))), + for ( + addr, + CollatedLeaf { + balance, + nonce, + code, + code_length, + storage, + }, + ) in leaves + { + use smt_trie::keys::{key_balance, key_code, key_code_length, key_nonce, key_storage}; + + for (value, key_fn) in [ + (balance, key_balance as fn(_) -> _), + (nonce, key_nonce), + (code, key_code), + (code_length, key_code_length), + ] { + if let Some(value) = value { + trie.set(key_fn(addr), value); + } + } + for (slot, value) in storage { + trie.set(key_storage(addr, slot), value); + } + } + Ok(trie) } -} -#[test] -fn test_tries() { for (ix, case) in serde_json::from_str::>(include_str!("cases/hermez_cdk_erigon.json")) .unwrap() @@ -234,10 +277,11 @@ fn test_tries() { { println!("case {}", ix); let instructions = crate::wire::parse(&case.bytes).unwrap(); - let frontend = frontend(instructions).unwrap(); + let (node, _code) = fold(instructions).unwrap(); + let trie = node2trie(node).unwrap(); assert_eq!(case.expected_state_root, { let mut it = [0; 32]; - smt_trie::utils::hashout2u(frontend.trie.root).to_big_endian(&mut it); + smt_trie::utils::hashout2u(trie.root).to_big_endian(&mut it); ethereum_types::H256(it) }); } diff --git a/trace_decoder/src/wire.rs b/trace_decoder/src/wire.rs index 6f56f1e44..63dee6040 100644 --- a/trace_decoder/src/wire.rs +++ b/trace_decoder/src/wire.rs @@ -1,6 +1,6 @@ //! We support two wire formats: -//! - Type 1, based on [this specification](https://gist.github.com/mandrigin/ff7eccf30d0ef9c572bafcb0ab665cff#the-bytes-layout). -//! - Type 2, loosely based on [this specification](https://github.com/0xPolygonHermez/cdk-erigon/blob/d1d6b3c7a4c81c46fd995c1baa5c1f8069ff0348/turbo/trie/WITNESS.md) +//! - Type 1 (AKA MPT), based on [this specification](https://gist.github.com/mandrigin/ff7eccf30d0ef9c572bafcb0ab665cff#the-bytes-layout). +//! - Type 2 (AKA SMT), loosely based on [this specification](https://github.com/0xPolygonHermez/cdk-erigon/blob/d1d6b3c7a4c81c46fd995c1baa5c1f8069ff0348/turbo/trie/WITNESS.md) //! //! Fortunately, their opcodes don't conflict, so we can have a single //! [`Instruction`] type, with shared parsing logic in this module, and bail on @@ -80,6 +80,8 @@ pub enum Instruction { } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/705 +// `address` and `value` should be fixed length fields pub struct SmtLeaf { pub node_type: SmtLeafType, pub address: NonEmpty>, @@ -87,6 +89,8 @@ pub struct SmtLeaf { } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/705 +// `Storage` should contain a fixed length field pub enum SmtLeafType { Balance, Nonce, diff --git a/trace_decoder/tests/consistent-with-header.rs b/trace_decoder/tests/consistent-with-header.rs index 609fd57bb..63df41b4f 100644 --- a/trace_decoder/tests/consistent-with-header.rs +++ b/trace_decoder/tests/consistent-with-header.rs @@ -11,6 +11,7 @@ use itertools::Itertools; use libtest_mimic::{Arguments, Trial}; use mpt_trie::partial_trie::PartialTrie as _; use trace_decoder::observer::DummyObserver; +use zero::prover::WIRE_DISPOSITION; fn main() -> anyhow::Result<()> { let mut trials = vec![]; @@ -29,6 +30,7 @@ fn main() -> anyhow::Result<()> { other.clone(), batch_size, &mut DummyObserver::new(), + WIRE_DISPOSITION, ) .map_err(|e| format!("{e:?}"))?; // get the full cause chain check!(gen_inputs.len() >= 2); diff --git a/trace_decoder/tests/simulate-execution.rs b/trace_decoder/tests/simulate-execution.rs index d0476c2b7..fc7136c34 100644 --- a/trace_decoder/tests/simulate-execution.rs +++ b/trace_decoder/tests/simulate-execution.rs @@ -9,6 +9,7 @@ use common::{cases, Case}; use libtest_mimic::{Arguments, Trial}; use plonky2::field::goldilocks_field::GoldilocksField; use trace_decoder::observer::DummyObserver; +use zero::prover::WIRE_DISPOSITION; fn main() -> anyhow::Result<()> { let mut trials = vec![]; @@ -20,11 +21,16 @@ fn main() -> anyhow::Result<()> { other, } in cases()? { - let gen_inputs = - trace_decoder::entrypoint(trace, other, batch_size, &mut DummyObserver::new()) - .context(format!( - "error in `trace_decoder` for {name} at batch size {batch_size}" - ))?; + let gen_inputs = trace_decoder::entrypoint( + trace, + other, + batch_size, + &mut DummyObserver::new(), + WIRE_DISPOSITION, + ) + .context(format!( + "error in `trace_decoder` for {name} at batch size {batch_size}" + ))?; for (ix, gi) in gen_inputs.into_iter().enumerate() { trials.push(Trial::test( format!("{name}@{batch_size}/{ix}"), diff --git a/zero/Cargo.toml b/zero/Cargo.toml index 22c2a8bfb..7cbf2f351 100644 --- a/zero/Cargo.toml +++ b/zero/Cargo.toml @@ -15,6 +15,7 @@ alloy-compat = "0.1.0" anyhow.workspace = true async-stream.workspace = true axum.workspace = true +cfg-if = "1.0.0" clap = { workspace = true, features = ["derive", "string"] } compat.workspace = true directories = "5.0.1" diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index d49cdde5c..164751df2 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -14,6 +14,7 @@ use url::Url; use zero::block_interval::BlockInterval; use zero::block_interval::BlockIntervalStream; use zero::prover::BlockProverInput; +use zero::prover::WIRE_DISPOSITION; use zero::provider::CachedProvider; use zero::rpc; @@ -172,6 +173,7 @@ impl Cli { block_prover_input.other_data, batch_size, &mut DummyObserver::new(), + WIRE_DISPOSITION, )?; if let Some(index) = tx_info.transaction_index { diff --git a/zero/src/bin/trie_diff.rs b/zero/src/bin/trie_diff.rs index 4c00d2ca3..c211cc528 100644 --- a/zero/src/bin/trie_diff.rs +++ b/zero/src/bin/trie_diff.rs @@ -26,6 +26,7 @@ use regex::Regex; use trace_decoder::observer::TriesObserver; use tracing::{error, info}; use zero::ops::register; +use zero::prover::WIRE_DISPOSITION; use zero::prover::{cli::CliProverConfig, BlockProverInput, ProverConfig}; /// This binary is a debugging tool used to compare @@ -97,6 +98,7 @@ async fn main() -> Result<()> { block_prover_input.other_data.clone(), prover_config.batch_size, &mut observer, + WIRE_DISPOSITION, )?; info!( "Number of collected batch tries for block {}: {}", diff --git a/zero/src/prover.rs b/zero/src/prover.rs index 4e221709c..7cc840f02 100644 --- a/zero/src/prover.rs +++ b/zero/src/prover.rs @@ -25,7 +25,7 @@ use tokio::io::AsyncWriteExt; use tokio::sync::mpsc::Receiver; use tokio::sync::{oneshot, Semaphore}; use trace_decoder::observer::DummyObserver; -use trace_decoder::{BlockTrace, OtherBlockData}; +use trace_decoder::{BlockTrace, OtherBlockData, WireDisposition}; use tracing::{error, info}; use crate::fs::generate_block_proof_file_name; @@ -55,6 +55,18 @@ pub struct ProofRuntime { // batches as soon as they are generated. static PARALLEL_BLOCK_PROVING_PERMIT_POOL: Semaphore = Semaphore::const_new(0); +pub const WIRE_DISPOSITION: WireDisposition = { + cfg_if::cfg_if! { + if #[cfg(feature = "eth_mainnet")] { + WireDisposition::Type1 + } else if #[cfg(feature = "cdk_erigon")] { + WireDisposition::Type2 + } else { + compile_error!("must select a feature"); + } + } +}; + #[derive(Debug, Clone)] pub struct ProverConfig { pub batch_size: usize, @@ -101,6 +113,7 @@ impl BlockProverInput { self.other_data, batch_size, &mut DummyObserver::new(), + WIRE_DISPOSITION, )?; // Create segment proof. @@ -193,6 +206,7 @@ impl BlockProverInput { self.other_data, batch_size, &mut DummyObserver::new(), + WIRE_DISPOSITION, )?; let seg_ops = ops::SegmentProofTestOnly { From 4f70195287b9bd1832bd92df119c81027333db96 Mon Sep 17 00:00:00 2001 From: Robin Salen <30937548+Nashtare@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:25:58 -0400 Subject: [PATCH 04/17] Bump plonky2 (serialization fix) (#729) * Bump plonky2 * Fix test and add serialization check * Add test * Move test to empty_tables * Apply comments * Revert "Apply comments" This reverts commit 50553fda7c4184c557bf48b56457d8edfd95fc85. * Tweak test * Pin to main branch --- Cargo.lock | 15 +- Cargo.toml | 6 +- evm_arithmetization/src/all_stark.rs | 19 ++ .../src/fixed_recursive_verifier.rs | 125 +++---- evm_arithmetization/src/recursive_verifier.rs | 322 +++++++++++------- evm_arithmetization/src/verifier.rs | 17 +- evm_arithmetization/src/witness/errors.rs | 7 + evm_arithmetization/tests/empty_tables.rs | 32 ++ evm_arithmetization/tests/two_to_one_block.rs | 2 +- 9 files changed, 347 insertions(+), 198 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ea4fb8060..ef891b1e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3644,7 +3644,7 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plonky2" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b#2488cdacd49ede15737bc1172546d82e9521b79b" dependencies = [ "ahash", "anyhow", @@ -3655,7 +3655,7 @@ dependencies = [ "log", "num", "plonky2_field", - "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee)", + "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b)", "plonky2_util", "rand", "rand_chacha", @@ -3668,7 +3668,7 @@ dependencies = [ [[package]] name = "plonky2_field" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b#2488cdacd49ede15737bc1172546d82e9521b79b" dependencies = [ "anyhow", "itertools 0.11.0", @@ -3692,7 +3692,7 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b#2488cdacd49ede15737bc1172546d82e9521b79b" dependencies = [ "rayon", ] @@ -3700,7 +3700,7 @@ dependencies = [ [[package]] name = "plonky2_util" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b#2488cdacd49ede15737bc1172546d82e9521b79b" [[package]] name = "plotters" @@ -4661,7 +4661,7 @@ checksum = "8acdd7dbfcfb5dd6e46c63512508bf71c2043f70b8f143813ad75cb5e8a589f2" [[package]] name = "starky" version = "0.4.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b#2488cdacd49ede15737bc1172546d82e9521b79b" dependencies = [ "ahash", "anyhow", @@ -4670,8 +4670,9 @@ dependencies = [ "log", "num-bigint", "plonky2", - "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee)", + "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b)", "plonky2_util", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e47c71386..3b038ff26 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,10 +111,10 @@ zk_evm_proc_macro = { path = "proc_macro", version = "0.1.0" } zero = { path = "zero", default-features = false } # plonky2-related dependencies -plonky2 = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } +plonky2 = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "2488cdacd49ede15737bc1172546d82e9521b79b" } plonky2_maybe_rayon = "0.2.0" -plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } -starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } +plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "2488cdacd49ede15737bc1172546d82e9521b79b" } +starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "2488cdacd49ede15737bc1172546d82e9521b79b" } [workspace.lints.clippy] too_long_first_doc_paragraph = "allow" diff --git a/evm_arithmetization/src/all_stark.rs b/evm_arithmetization/src/all_stark.rs index 0ec87f3db..0be307db5 100644 --- a/evm_arithmetization/src/all_stark.rs +++ b/evm_arithmetization/src/all_stark.rs @@ -129,6 +129,11 @@ impl Table { } } +/// The total number of CTLs used by the zkEVM. +pub(crate) const NUM_CTLS: usize = if cfg!(feature = "cdk_erigon") { 13 } else { 10 }; +/// The position of the Memory CTL within all CTLs of the zkEVM. +pub(crate) const MEMORY_CTL_IDX: usize = 6; + /// Returns all the `CrossTableLookups` used for proving the EVM. pub(crate) fn all_cross_table_lookups() -> Vec> { vec![ @@ -419,3 +424,17 @@ fn ctl_poseidon_general_output() -> CrossTableLookup { poseidon_stark::ctl_looked_general_output(), ) } + +#[cfg(test)] +mod tests { + use plonky2::field::goldilocks_field::GoldilocksField; + + use super::*; + + type F = GoldilocksField; + + #[test] + fn check_num_ctls() { + assert_eq!(all_cross_table_lookups::().len(), NUM_CTLS); + } +} diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 96ecce6d3..6521724a7 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -37,7 +37,8 @@ use starky::proof::StarkProofWithMetadata; use starky::stark::Stark; use crate::all_stark::{ - all_cross_table_lookups, AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES, + all_cross_table_lookups, AllStark, Table, KECCAK_TABLES_INDICES, MEMORY_CTL_IDX, NUM_CTLS, + NUM_TABLES, }; use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::segments::{GenerationSegmentData, SegmentDataIterator}; @@ -998,19 +999,23 @@ where // Extra sums to add to the looked last value. // Only necessary for the Memory values. - let mut extra_looking_sums = - vec![vec![builder.zero(); stark_config.num_challenges]; NUM_TABLES]; + let mut extra_looking_sums = HashMap::from_iter( + (0..NUM_CTLS).map(|i| (i, vec![builder.zero(); stark_config.num_challenges])), + ); // Memory - extra_looking_sums[*Table::Memory] = (0..stark_config.num_challenges) - .map(|c| { - get_memory_extra_looking_sum_circuit( - &mut builder, - &public_values, - ctl_challenges.challenges[c], - ) - }) - .collect_vec(); + extra_looking_sums.insert( + MEMORY_CTL_IDX, + (0..stark_config.num_challenges) + .map(|c| { + get_memory_extra_looking_sum_circuit( + &mut builder, + &public_values, + ctl_challenges.challenges[c], + ) + }) + .collect_vec(), + ); // Ensure that when Keccak tables are skipped, the Keccak tables' ctl_zs_first // are all zeros. @@ -1026,7 +1031,7 @@ where &mut builder, all_cross_table_lookups(), pis.map(|p| p.ctl_zs_first), - Some(&extra_looking_sums), + &extra_looking_sums, stark_config, ); @@ -2013,11 +2018,11 @@ where let dummy_proof_data = self.table_dummy_proofs[table] .as_ref() .ok_or_else(|| anyhow::format_err!("No dummy_proof_data"))?; - root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO); + root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO)?; root_inputs.set_proof_with_pis_target( &self.root.proof_with_pis[table], &dummy_proof_data.proof, - ); + )?; } else { let stark_proof = &all_proof.multi_proof.stark_proofs[table] .as_ref() @@ -2042,9 +2047,9 @@ where root_inputs.set_target( self.root.index_verifier_data[table], F::from_canonical_usize(index_verifier_data), - ); + )?; root_inputs - .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof)?; } check_abort_signal(abort_signal.clone())?; @@ -2053,7 +2058,7 @@ where root_inputs.set_verifier_data_target( &self.root.cyclic_vk, &self.segment_aggregation.circuit.verifier_only, - ); + )?; set_public_value_targets( &mut root_inputs, @@ -2064,7 +2069,7 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; - root_inputs.set_bool_target(self.root.use_keccak_tables, all_proof.use_keccak_tables); + root_inputs.set_bool_target(self.root.use_keccak_tables, all_proof.use_keccak_tables)?; let root_proof = self.root.circuit.prove(root_inputs)?; @@ -2141,11 +2146,11 @@ where let dummy_proof = self.table_dummy_proofs[table] .as_ref() .ok_or_else(|| anyhow::format_err!("Unable to get dummpy proof"))?; - root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO); + root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO)?; root_inputs.set_proof_with_pis_target( &self.root.proof_with_pis[table], &dummy_proof.proof, - ); + )?; } else { let (table_circuit, index_verifier_data) = &table_circuits[table] .as_ref() @@ -2153,14 +2158,14 @@ where root_inputs.set_target( self.root.index_verifier_data[table], F::from_canonical_u8(*index_verifier_data), - ); + )?; let stark_proof = all_proof.multi_proof.stark_proofs[table] .as_ref() .ok_or_else(|| anyhow::format_err!("Unable to get stark proof"))?; let shrunk_proof = table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; root_inputs - .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof)?; } check_abort_signal(abort_signal.clone())?; @@ -2169,7 +2174,7 @@ where root_inputs.set_verifier_data_target( &self.root.cyclic_vk, &self.segment_aggregation.circuit.verifier_only, - ); + )?; set_public_value_targets( &mut root_inputs, @@ -2180,7 +2185,7 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; - root_inputs.set_bool_target(self.root.use_keccak_tables, all_proof.use_keccak_tables); + root_inputs.set_bool_target(self.root.use_keccak_tables, all_proof.use_keccak_tables)?; let root_proof = self.root.circuit.prove(root_inputs)?; @@ -2229,7 +2234,7 @@ where &self.segment_aggregation.circuit, &mut agg_inputs, lhs_proof, - ); + )?; // If rhs is dummy, the rhs proof is also set to be the lhs. let real_rhs_proof = if rhs_is_dummy { lhs_proof } else { rhs_proof }; @@ -2241,12 +2246,12 @@ where &self.segment_aggregation.circuit, &mut agg_inputs, real_rhs_proof, - ); + )?; agg_inputs.set_verifier_data_target( &self.segment_aggregation.cyclic_vk, &self.segment_aggregation.circuit.verifier_only, - ); + )?; // Aggregates both `PublicValues` from the provided proofs into a single one. let lhs_public_values = &lhs.proof_with_pvs.public_values; @@ -2339,7 +2344,7 @@ where &self.batch_aggregation.circuit, &mut batch_inputs, &lhs.intern, - ); + )?; Self::set_dummy_if_necessary( &self.batch_aggregation.rhs, @@ -2347,12 +2352,12 @@ where &self.batch_aggregation.circuit, &mut batch_inputs, &rhs.intern, - ); + )?; batch_inputs.set_verifier_data_target( &self.batch_aggregation.cyclic_vk, &self.batch_aggregation.circuit.verifier_only, - ); + )?; let lhs_pvs = &lhs.public_values; let batch_public_values = PublicValues { @@ -2393,20 +2398,20 @@ where circuit: &CircuitData, agg_inputs: &mut PartialWitness, proof: &ProofWithPublicInputs, - ) { - agg_inputs.set_bool_target(agg_child.is_agg, is_agg); - agg_inputs.set_bool_target(agg_child.is_dummy, is_dummy); + ) -> anyhow::Result<()> { + agg_inputs.set_bool_target(agg_child.is_agg, is_agg)?; + agg_inputs.set_bool_target(agg_child.is_dummy, is_dummy)?; if is_agg { - agg_inputs.set_proof_with_pis_target(&agg_child.agg_proof, proof); + agg_inputs.set_proof_with_pis_target(&agg_child.agg_proof, proof)?; } else { Self::set_dummy_proof_with_cyclic_vk_pis( circuit, agg_inputs, &agg_child.agg_proof, proof, - ); + )?; } - agg_inputs.set_proof_with_pis_target(&agg_child.real_proof, proof); + agg_inputs.set_proof_with_pis_target(&agg_child.real_proof, proof) } /// Create a final block proof, once all transactions of a given block have @@ -2438,10 +2443,10 @@ where block_inputs.set_bool_target( self.block.has_parent_block, opt_parent_block_proof.is_some(), - ); + )?; if let Some(parent_block_proof) = opt_parent_block_proof { block_inputs - .set_proof_with_pis_target(&self.block.parent_block_proof, parent_block_proof); + .set_proof_with_pis_target(&self.block.parent_block_proof, parent_block_proof)?; } else { if agg_root_proof.public_values.trie_roots_before.state_root != agg_root_proof @@ -2584,13 +2589,14 @@ where &self.block.circuit.verifier_only, nonzero_pis, ), - ); + )?; } - block_inputs.set_proof_with_pis_target(&self.block.agg_root_proof, &agg_root_proof.intern); + block_inputs + .set_proof_with_pis_target(&self.block.agg_root_proof, &agg_root_proof.intern)?; block_inputs - .set_verifier_data_target(&self.block.cyclic_vk, &self.block.circuit.verifier_only); + .set_verifier_data_target(&self.block.cyclic_vk, &self.block.circuit.verifier_only)?; // This is basically identical to this block public values, apart from the // `trie_roots_before` that may come from the previous proof, if any. @@ -2649,13 +2655,15 @@ where )> { let mut block_wrapper_inputs = PartialWitness::new(); - block_wrapper_inputs - .set_proof_with_pis_target(&self.block_wrapper.parent_block_proof, &block_proof.intern); + block_wrapper_inputs.set_proof_with_pis_target( + &self.block_wrapper.parent_block_proof, + &block_proof.intern, + )?; block_wrapper_inputs.set_verifier_data_target( &self.block_wrapper.cyclic_vk, // dummy &self.block_wrapper.circuit.verifier_only, - ); + )?; let final_pvs = block_proof.public_values.clone().into(); set_final_public_value_targets( @@ -2709,7 +2717,7 @@ where &self.two_to_one_block.circuit, &mut witness, lhs, - ); + )?; Self::set_dummy_if_necessary( &self.two_to_one_block.rhs, @@ -2717,15 +2725,14 @@ where &self.two_to_one_block.circuit, &mut witness, rhs, - ); + )?; witness.set_verifier_data_target( &self.two_to_one_block.cyclic_vk, &self.two_to_one_block.circuit.verifier_only, - ); + )?; - let proof = self.two_to_one_block.circuit.prove(witness)?; - Ok(proof) + self.two_to_one_block.circuit.prove(witness) } /// Verifies an existing block aggregation proof. @@ -2756,7 +2763,7 @@ where witness: &mut PartialWitness, agg_proof_with_pis: &ProofWithPublicInputsTarget, base_proof_with_pis: &ProofWithPublicInputs, - ) { + ) -> anyhow::Result<()> { let ProofWithPublicInputs { proof: base_proof, public_inputs: _, @@ -2767,7 +2774,7 @@ where } = agg_proof_with_pis; // The proof remains the same. - witness.set_proof_target(agg_proof_targets, base_proof); + witness.set_proof_target(agg_proof_targets, base_proof)?; let cyclic_verifying_data = &circuit_agg.verifier_only; let mut cyclic_vk = cyclic_verifying_data.circuit_digest.to_vec(); @@ -2778,8 +2785,10 @@ where // Set dummy public inputs. for (&pi_t, pi) in agg_pi_targets.iter().zip_eq(dummy_pis) { - witness.set_target(pi_t, pi); + witness.set_target(pi_t, pi)?; } + + Ok(()) } /// If the [`AggregationChild`] is a base proof and not an aggregation @@ -2794,19 +2803,19 @@ where circuit: &CircuitData, agg_inputs: &mut PartialWitness, proof: &ProofWithPublicInputs, - ) { - agg_inputs.set_bool_target(agg_child.is_agg, is_agg); + ) -> anyhow::Result<()> { + agg_inputs.set_bool_target(agg_child.is_agg, is_agg)?; if is_agg { - agg_inputs.set_proof_with_pis_target(&agg_child.agg_proof, proof); + agg_inputs.set_proof_with_pis_target(&agg_child.agg_proof, proof)?; } else { Self::set_dummy_proof_with_cyclic_vk_pis( circuit, agg_inputs, &agg_child.agg_proof, proof, - ); + )?; } - agg_inputs.set_proof_with_pis_target(&agg_child.base_proof, proof); + agg_inputs.set_proof_with_pis_target(&agg_child.base_proof, proof) } } diff --git a/evm_arithmetization/src/recursive_verifier.rs b/evm_arithmetization/src/recursive_verifier.rs index 14ed5957e..f70b9f166 100644 --- a/evm_arithmetization/src/recursive_verifier.rs +++ b/evm_arithmetization/src/recursive_verifier.rs @@ -156,7 +156,7 @@ where &self.stark_proof_target, &proof_with_metadata.proof, self.zero_target, - ); + )?; for (challenge_target, challenge) in self .ctl_challenges_target @@ -164,14 +164,14 @@ where .iter() .zip(&ctl_challenges.challenges) { - inputs.set_target(challenge_target.beta, challenge.beta); - inputs.set_target(challenge_target.gamma, challenge.gamma); + inputs.set_target(challenge_target.beta, challenge.beta)?; + inputs.set_target(challenge_target.gamma, challenge.gamma)?; } inputs.set_target_arr( self.init_challenger_state_target.as_ref(), proof_with_metadata.init_challenger_state.as_ref(), - ); + )?; self.circuit.prove(inputs) } @@ -199,7 +199,7 @@ where proof: &ProofWithPublicInputs, ) -> Result> { let mut inputs = PartialWitness::new(); - inputs.set_proof_with_pis_target(&self.proof_with_pis_target, proof); + inputs.set_proof_with_pis_target(&self.proof_with_pis_target, proof)?; self.circuit.prove(inputs) } } @@ -842,12 +842,12 @@ where witness, &public_values_target.trie_roots_before, &public_values.trie_roots_before, - ); + )?; set_trie_roots_target( witness, &public_values_target.trie_roots_after, &public_values.trie_roots_after, - ); + )?; set_block_metadata_target( witness, &public_values_target.block_metadata, @@ -857,7 +857,7 @@ where witness, &public_values_target.block_hashes, &public_values.block_hashes, - ); + )?; set_extra_public_values_target( witness, &public_values_target.extra_block_data, @@ -906,10 +906,12 @@ where H: Hasher, W: Witness, { - witness.set_target( - public_values_target.chain_id, - F::from_canonical_u64(public_values.chain_id.low_u64()), - ); + witness + .set_target( + public_values_target.chain_id, + F::from_canonical_u64(public_values.chain_id.low_u64()), + ) + .map_err(ProgramError::from)?; for (i, limb) in public_values .checkpoint_state_trie_root @@ -918,14 +920,18 @@ where .into_iter() .enumerate() { - witness.set_target( - public_values_target.checkpoint_state_trie_root[2 * i], - F::from_canonical_u32(limb as u32), - ); - witness.set_target( - public_values_target.checkpoint_state_trie_root[2 * i + 1], - F::from_canonical_u32((limb >> 32) as u32), - ); + witness + .set_target( + public_values_target.checkpoint_state_trie_root[2 * i], + F::from_canonical_u32(limb as u32), + ) + .map_err(ProgramError::from)?; + witness + .set_target( + public_values_target.checkpoint_state_trie_root[2 * i + 1], + F::from_canonical_u32((limb >> 32) as u32), + ) + .map_err(ProgramError::from)?; } for (i, limb) in public_values @@ -935,18 +941,24 @@ where .into_iter() .enumerate() { - witness.set_target( - public_values_target.new_state_trie_root[2 * i], - F::from_canonical_u32(limb as u32), - ); - witness.set_target( - public_values_target.new_state_trie_root[2 * i + 1], - F::from_canonical_u32((limb >> 32) as u32), - ); + witness + .set_target( + public_values_target.new_state_trie_root[2 * i], + F::from_canonical_u32(limb as u32), + ) + .map_err(ProgramError::from)?; + witness + .set_target( + public_values_target.new_state_trie_root[2 * i + 1], + F::from_canonical_u32((limb >> 32) as u32), + ) + .map_err(ProgramError::from)?; } for (i, limb) in public_values.new_consolidated_hash.iter().enumerate() { - witness.set_target(public_values_target.new_consolidated_hash[i], *limb); + witness + .set_target(public_values_target.new_consolidated_hash[i], *limb) + .map_err(ProgramError::from)?; } Ok(()) @@ -956,7 +968,8 @@ pub(crate) fn set_trie_roots_target( witness: &mut W, trie_roots_target: &TrieRootsTarget, trie_roots: &TrieRoots, -) where +) -> Result<()> +where F: RichField + Extendable, W: Witness, { @@ -964,11 +977,11 @@ pub(crate) fn set_trie_roots_target( witness.set_target( trie_roots_target.state_root[2 * i], F::from_canonical_u32(limb as u32), - ); + )?; witness.set_target( trie_roots_target.state_root[2 * i + 1], F::from_canonical_u32((limb >> 32) as u32), - ); + )?; } for (i, limb) in trie_roots @@ -981,11 +994,11 @@ pub(crate) fn set_trie_roots_target( witness.set_target( trie_roots_target.transactions_root[2 * i], F::from_canonical_u32(limb as u32), - ); + )?; witness.set_target( trie_roots_target.transactions_root[2 * i + 1], F::from_canonical_u32((limb >> 32) as u32), - ); + )?; } for (i, limb) in trie_roots @@ -998,12 +1011,14 @@ pub(crate) fn set_trie_roots_target( witness.set_target( trie_roots_target.receipts_root[2 * i], F::from_canonical_u32(limb as u32), - ); + )?; witness.set_target( trie_roots_target.receipts_root[2 * i + 1], F::from_canonical_u32((limb >> 32) as u32), - ); + )?; } + + Ok(()) } #[cfg(feature = "cdk_erigon")] @@ -1019,7 +1034,9 @@ where match burn_addr_target { BurnAddrTarget::BurnAddr(addr_target) => { let burn_addr_limbs: [F; 8] = u256_limbs::(burn_addr); - witness.set_target_arr(addr_target, &burn_addr_limbs); + witness + .set_target_arr(addr_target, &burn_addr_limbs) + .map_err(ProgramError::from)?; } BurnAddrTarget::Burnt() => panic!("There should be an address target set in cdk_erigon."), } @@ -1040,73 +1057,105 @@ where u256_limbs::(U256::from_big_endian(&block_metadata.block_beneficiary.0))[..5] .try_into() .unwrap(); - witness.set_target_arr(&block_metadata_target.block_beneficiary, &beneficiary_limbs); - witness.set_target( - block_metadata_target.block_timestamp, - u256_to_u32(block_metadata.block_timestamp)?, - ); - witness.set_target( - block_metadata_target.block_number, - u256_to_u32(block_metadata.block_number)?, - ); - witness.set_target( - block_metadata_target.block_difficulty, - u256_to_u32(block_metadata.block_difficulty)?, - ); - witness.set_target_arr( - &block_metadata_target.block_random, - &h256_limbs(block_metadata.block_random), - ); - witness.set_target( - block_metadata_target.block_gaslimit, - u256_to_u32(block_metadata.block_gaslimit)?, - ); - witness.set_target( - block_metadata_target.block_chain_id, - u256_to_u32(block_metadata.block_chain_id)?, - ); + witness + .set_target_arr(&block_metadata_target.block_beneficiary, &beneficiary_limbs) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_timestamp, + u256_to_u32(block_metadata.block_timestamp)?, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_number, + u256_to_u32(block_metadata.block_number)?, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_difficulty, + u256_to_u32(block_metadata.block_difficulty)?, + ) + .map_err(ProgramError::from)?; + witness + .set_target_arr( + &block_metadata_target.block_random, + &h256_limbs(block_metadata.block_random), + ) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_gaslimit, + u256_to_u32(block_metadata.block_gaslimit)?, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_chain_id, + u256_to_u32(block_metadata.block_chain_id)?, + ) + .map_err(ProgramError::from)?; // Basefee fits in 2 limbs let basefee = u256_to_u64(block_metadata.block_base_fee)?; - witness.set_target(block_metadata_target.block_base_fee[0], basefee.0); - witness.set_target(block_metadata_target.block_base_fee[1], basefee.1); - witness.set_target( - block_metadata_target.block_gas_used, - u256_to_u32(block_metadata.block_gas_used)?, - ); + witness + .set_target(block_metadata_target.block_base_fee[0], basefee.0) + .map_err(ProgramError::from)?; + witness + .set_target(block_metadata_target.block_base_fee[1], basefee.1) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_gas_used, + u256_to_u32(block_metadata.block_gas_used)?, + ) + .map_err(ProgramError::from)?; #[cfg(feature = "eth_mainnet")] { // BlobGasUsed fits in 2 limbs let blob_gas_used = u256_to_u64(block_metadata.block_blob_gas_used)?; - witness.set_target( - block_metadata_target.block_blob_gas_used[0], - blob_gas_used.0, - ); - witness.set_target( - block_metadata_target.block_blob_gas_used[1], - blob_gas_used.1, - ); + witness + .set_target( + block_metadata_target.block_blob_gas_used[0], + blob_gas_used.0, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_blob_gas_used[1], + blob_gas_used.1, + ) + .map_err(ProgramError::from)?; // ExcessBlobGas fits in 2 limbs let excess_blob_gas = u256_to_u64(block_metadata.block_excess_blob_gas)?; - witness.set_target( - block_metadata_target.block_excess_blob_gas[0], - excess_blob_gas.0, - ); - witness.set_target( - block_metadata_target.block_excess_blob_gas[1], - excess_blob_gas.1, - ); - - witness.set_target_arr( - &block_metadata_target.parent_beacon_block_root, - &h256_limbs(block_metadata.parent_beacon_block_root), - ); + witness + .set_target( + block_metadata_target.block_excess_blob_gas[0], + excess_blob_gas.0, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_excess_blob_gas[1], + excess_blob_gas.1, + ) + .map_err(ProgramError::from)?; + + witness + .set_target_arr( + &block_metadata_target.parent_beacon_block_root, + &h256_limbs(block_metadata.parent_beacon_block_root), + ) + .map_err(ProgramError::from)?; } let mut block_bloom_limbs = [F::ZERO; 64]; for (i, limbs) in block_bloom_limbs.chunks_exact_mut(8).enumerate() { limbs.copy_from_slice(&u256_limbs(block_metadata.block_bloom[i])); } - witness.set_target_arr(&block_metadata_target.block_bloom, &block_bloom_limbs); + witness + .set_target_arr(&block_metadata_target.block_bloom, &block_bloom_limbs) + .map_err(ProgramError::from)?; Ok(()) } @@ -1115,7 +1164,8 @@ pub(crate) fn set_block_hashes_target( witness: &mut W, block_hashes_target: &BlockHashesTarget, block_hashes: &BlockHashes, -) where +) -> Result<()> +where F: RichField + Extendable, W: Witness, { @@ -1124,10 +1174,10 @@ pub(crate) fn set_block_hashes_target( witness.set_target_arr( &block_hashes_target.prev_hashes[8 * i..8 * (i + 1)], &block_hash_limbs, - ); + )?; } let cur_block_hash_limbs: [F; 8] = h256_limbs::(block_hashes.cur_hash); - witness.set_target_arr(&block_hashes_target.cur_hash, &cur_block_hash_limbs); + witness.set_target_arr(&block_hashes_target.cur_hash, &cur_block_hash_limbs) } pub(crate) fn set_extra_public_values_target( @@ -1139,24 +1189,36 @@ where F: RichField + Extendable, W: Witness, { - witness.set_target_arr( - &ed_target.checkpoint_state_trie_root, - &h256_limbs::(ed.checkpoint_state_trie_root), - ); - witness.set_target_arr( - &ed_target.checkpoint_consolidated_hash, - &ed.checkpoint_consolidated_hash, - ); - witness.set_target( - ed_target.txn_number_before, - u256_to_u32(ed.txn_number_before)?, - ); - witness.set_target( - ed_target.txn_number_after, - u256_to_u32(ed.txn_number_after)?, - ); - witness.set_target(ed_target.gas_used_before, u256_to_u32(ed.gas_used_before)?); - witness.set_target(ed_target.gas_used_after, u256_to_u32(ed.gas_used_after)?); + witness + .set_target_arr( + &ed_target.checkpoint_state_trie_root, + &h256_limbs::(ed.checkpoint_state_trie_root), + ) + .map_err(ProgramError::from)?; + witness + .set_target_arr( + &ed_target.checkpoint_consolidated_hash, + &ed.checkpoint_consolidated_hash, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + ed_target.txn_number_before, + u256_to_u32(ed.txn_number_before)?, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + ed_target.txn_number_after, + u256_to_u32(ed.txn_number_after)?, + ) + .map_err(ProgramError::from)?; + witness + .set_target(ed_target.gas_used_before, u256_to_u32(ed.gas_used_before)?) + .map_err(ProgramError::from)?; + witness + .set_target(ed_target.gas_used_after, u256_to_u32(ed.gas_used_after)?) + .map_err(ProgramError::from)?; Ok(()) } @@ -1170,12 +1232,24 @@ where F: RichField + Extendable, W: Witness, { - witness.set_target(rd_target.program_counter, u256_to_u32(rd.program_counter)?); - witness.set_target(rd_target.is_kernel, u256_to_u32(rd.is_kernel)?); - witness.set_target(rd_target.stack_len, u256_to_u32(rd.stack_len)?); - witness.set_target_arr(&rd_target.stack_top, &u256_limbs(rd.stack_top)); - witness.set_target(rd_target.context, u256_to_u32(rd.context)?); - witness.set_target(rd_target.gas_used, u256_to_u32(rd.gas_used)?); + witness + .set_target(rd_target.program_counter, u256_to_u32(rd.program_counter)?) + .map_err(ProgramError::from)?; + witness + .set_target(rd_target.is_kernel, u256_to_u32(rd.is_kernel)?) + .map_err(ProgramError::from)?; + witness + .set_target(rd_target.stack_len, u256_to_u32(rd.stack_len)?) + .map_err(ProgramError::from)?; + witness + .set_target_arr(&rd_target.stack_top, &u256_limbs(rd.stack_top)) + .map_err(ProgramError::from)?; + witness + .set_target(rd_target.context, u256_to_u32(rd.context)?) + .map_err(ProgramError::from)?; + witness + .set_target(rd_target.gas_used, u256_to_u32(rd.gas_used)?) + .map_err(ProgramError::from)?; Ok(()) } @@ -1190,12 +1264,14 @@ where W: Witness, { for i in 0..mc.mem_cap.len() { - witness.set_hash_target( - mc_target.mem_cap.0[i], - HashOut { - elements: mc.mem_cap[i].map(|elt| F::from_canonical_u64(elt.as_u64())), - }, - ); + witness + .set_hash_target( + mc_target.mem_cap.0[i], + HashOut { + elements: mc.mem_cap[i].map(|elt| F::from_canonical_u64(elt.as_u64())), + }, + ) + .map_err(ProgramError::from)?; } Ok(()) } diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 81e9b502b..e7b0c5198 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -80,6 +80,7 @@ pub(crate) fn initial_memory_merkle_cap< pub mod testing { use anyhow::{ensure, Result}; use ethereum_types::{BigEndianHash, U256}; + use hashbrown::HashMap; use itertools::Itertools; use plonky2::field::extension::Extendable; use plonky2::hash::hash_types::RichField; @@ -92,7 +93,7 @@ pub mod testing { use starky::stark::Stark; use starky::verifier::verify_stark_proof_with_challenges; - use crate::all_stark::Table; + use crate::all_stark::{Table, MEMORY_CTL_IDX, NUM_CTLS}; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; use crate::get_challenges::testing::AllProofChallenges; @@ -249,12 +250,16 @@ pub mod testing { // Extra sums to add to the looked last value. // Only necessary for the Memory values. - let mut extra_looking_sums = vec![vec![F::ZERO; config.num_challenges]; NUM_TABLES]; + let mut extra_looking_sums = + HashMap::from_iter((0..NUM_CTLS).map(|i| (i, vec![F::ZERO; config.num_challenges]))); // Memory - extra_looking_sums[*Table::Memory] = (0..config.num_challenges) - .map(|i| get_memory_extra_looking_sum(&public_values, ctl_challenges.challenges[i])) - .collect_vec(); + extra_looking_sums.insert( + MEMORY_CTL_IDX, + (0..config.num_challenges) + .map(|i| get_memory_extra_looking_sum(&public_values, ctl_challenges.challenges[i])) + .collect_vec(), + ); let all_ctls = &all_stark.cross_table_lookups; @@ -289,7 +294,7 @@ pub mod testing { panic!("Unable to find stark_proof"); } }), - Some(&extra_looking_sums), + &extra_looking_sums, config, ) } diff --git a/evm_arithmetization/src/witness/errors.rs b/evm_arithmetization/src/witness/errors.rs index ef1e9f73b..adb9dd94d 100644 --- a/evm_arithmetization/src/witness/errors.rs +++ b/evm_arithmetization/src/witness/errors.rs @@ -16,6 +16,13 @@ pub enum ProgramError { IntegerTooLarge, ProverInputError(ProverInputError), UnknownContractCode, + Other(String), +} + +impl From for ProgramError { + fn from(e: anyhow::Error) -> Self { + ProgramError::Other(e.to_string()) + } } #[allow(clippy::enum_variant_names)] diff --git a/evm_arithmetization/tests/empty_tables.rs b/evm_arithmetization/tests/empty_tables.rs index d25901e24..e99c48731 100644 --- a/evm_arithmetization/tests/empty_tables.rs +++ b/evm_arithmetization/tests/empty_tables.rs @@ -1,5 +1,7 @@ #![cfg(feature = "eth_mainnet")] +use std::time::Duration; + use evm_arithmetization::fixed_recursive_verifier::AllRecursiveCircuits; use evm_arithmetization::prover::prove; use evm_arithmetization::testing_utils::{init_logger, segment_with_empty_tables}; @@ -8,6 +10,7 @@ use evm_arithmetization::AllStark; use plonky2::field::goldilocks_field::GoldilocksField; use plonky2::plonk::config::PoseidonGoldilocksConfig; use plonky2::timed; +use plonky2::util::serialization::{DefaultGateSerializer, DefaultGeneratorSerializer}; use plonky2::util::timing::TimingTree; use starky::config::StarkConfig; @@ -58,6 +61,7 @@ fn empty_tables() -> anyhow::Result<()> { &config, ) ); + let segment_proof = timed!( timing, log::Level::Info, @@ -76,5 +80,33 @@ fn empty_tables() -> anyhow::Result<()> { // Print timing details timing.print(); + // Test serialization of preprocessed circuits + { + let gate_serializer = DefaultGateSerializer; + let generator_serializer = DefaultGeneratorSerializer::::default(); + + let timing = TimingTree::new("serialize AllRecursiveCircuits", log::Level::Info); + let all_circuits_bytes = all_circuits + .to_bytes(false, &gate_serializer, &generator_serializer) + .map_err(|_| anyhow::Error::msg("AllRecursiveCircuits serialization failed."))?; + timing.filter(Duration::from_millis(100)).print(); + log::info!( + "AllRecursiveCircuits length: {} bytes", + all_circuits_bytes.len() + ); + + let timing = TimingTree::new("deserialize AllRecursiveCircuits", log::Level::Info); + let all_circuits_from_bytes = AllRecursiveCircuits::from_bytes( + &all_circuits_bytes, + false, + &gate_serializer, + &generator_serializer, + ) + .map_err(|_| anyhow::Error::msg("AllRecursiveCircuits deserialization failed."))?; + timing.filter(Duration::from_millis(100)).print(); + + assert_eq!(all_circuits, all_circuits_from_bytes); + } + Ok(()) } diff --git a/evm_arithmetization/tests/two_to_one_block.rs b/evm_arithmetization/tests/two_to_one_block.rs index 378aaf3f8..e808a33e4 100644 --- a/evm_arithmetization/tests/two_to_one_block.rs +++ b/evm_arithmetization/tests/two_to_one_block.rs @@ -172,7 +172,7 @@ fn test_two_to_one_block_aggregation() -> anyhow::Result<()> { let all_circuits = AllRecursiveCircuits::new( &all_stark, - &[16..17, 8..9, 12..13, 8..9, 8..9, 6..7, 17..18, 17..18, 7..8], + &[16..17, 8..9, 12..13, 8..9, 8..9, 6..7, 17..18, 16..17, 7..8], &config, ); From ce0ab8c861cb220d4008ac3d6b1844e76286f433 Mon Sep 17 00:00:00 2001 From: Sai <135601871+sai-deng@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:26:39 +0000 Subject: [PATCH 05/17] Fix max_cpu_len_log (#714) * fix * fix * fix * address comments --- evm_arithmetization/src/generation/state.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index 47b580f02..abe4b4f1a 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -188,6 +188,15 @@ pub(crate) trait State { { let halt_offsets = self.get_halt_offsets(); + if let Some(max_len_log) = max_cpu_len_log { + assert!( + (1 << max_len_log) >= NUM_EXTRA_CYCLES_AFTER, + "Target length (2^{}) is less than NUM_EXTRA_CYCLES_AFTER ({})", + max_len_log, + NUM_EXTRA_CYCLES_AFTER + ); + } + let cycle_limit = max_cpu_len_log.map(|max_len_log| (1 << max_len_log) - NUM_EXTRA_CYCLES_AFTER); From ae6c3c46935194c5c3c9af9c413dd704b8d22dee Mon Sep 17 00:00:00 2001 From: Sai <135601871+sai-deng@users.noreply.github.com> Date: Fri, 18 Oct 2024 18:21:14 +0000 Subject: [PATCH 06/17] feat: Enable more optional tables (#724) This PR enables BytePacking, Logic, MemAfter, and Poseidon tables as optional tables in the root circuit, reducing table proving time and recursion time. --- evm_arithmetization/src/all_stark.rs | 21 ++- .../src/fixed_recursive_verifier.rs | 148 ++++++++++------- evm_arithmetization/src/generation/mod.rs | 37 ++++- evm_arithmetization/src/get_challenges.rs | 4 +- evm_arithmetization/src/proof.rs | 24 ++- evm_arithmetization/src/prover.rs | 151 ++++++++---------- evm_arithmetization/src/testing_utils.rs | 6 +- evm_arithmetization/src/verifier.rs | 143 +++++++++-------- 8 files changed, 308 insertions(+), 226 deletions(-) diff --git a/evm_arithmetization/src/all_stark.rs b/evm_arithmetization/src/all_stark.rs index 0be307db5..41f4010fe 100644 --- a/evm_arithmetization/src/all_stark.rs +++ b/evm_arithmetization/src/all_stark.rs @@ -106,9 +106,24 @@ pub const NUM_TABLES: usize = if cfg!(feature = "cdk_erigon") { Table::MemAfter as usize + 1 }; -/// Indices of Keccak Tables -pub const KECCAK_TABLES_INDICES: [usize; 2] = - [Table::Keccak as usize, Table::KeccakSponge as usize]; +/// Indices of optional Tables +#[cfg(not(feature = "cdk_erigon"))] +pub const OPTIONAL_TABLE_INDICES: [usize; 5] = [ + Table::BytePacking as usize, + Table::Keccak as usize, + Table::KeccakSponge as usize, + Table::Logic as usize, + Table::MemAfter as usize, +]; +#[cfg(feature = "cdk_erigon")] +pub const OPTIONAL_TABLE_INDICES: [usize; 6] = [ + Table::BytePacking as usize, + Table::Keccak as usize, + Table::KeccakSponge as usize, + Table::Logic as usize, + Table::MemAfter as usize, + Table::Poseidon as usize, +]; impl Table { /// Returns all STARK table indices. diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 6521724a7..adde60d1b 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -37,8 +37,8 @@ use starky::proof::StarkProofWithMetadata; use starky::stark::Stark; use crate::all_stark::{ - all_cross_table_lookups, AllStark, Table, KECCAK_TABLES_INDICES, MEMORY_CTL_IDX, NUM_CTLS, - NUM_TABLES, + all_cross_table_lookups, AllStark, Table, MEMORY_CTL_IDX, NUM_CTLS, NUM_TABLES, + OPTIONAL_TABLE_INDICES, }; use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::segments::{GenerationSegmentData, SegmentDataIterator}; @@ -156,8 +156,8 @@ where /// for EVM root proofs; the circuit has them just to match the /// structure of aggregation proofs. cyclic_vk: VerifierCircuitTarget, - /// We can skip verifying Keccak tables when they are not in use. - use_keccak_tables: BoolTarget, + /// We can skip verifying tables when they are not in use. + table_in_use: [BoolTarget; NUM_TABLES], } impl RootCircuitData @@ -180,7 +180,9 @@ where } self.public_values.to_buffer(buffer)?; buffer.write_target_verifier_circuit(&self.cyclic_vk)?; - buffer.write_target_bool(self.use_keccak_tables)?; + for table_in_use in self.table_in_use { + buffer.write_target_bool(table_in_use)?; + } Ok(()) } @@ -200,7 +202,10 @@ where } let public_values = PublicValuesTarget::from_buffer(buffer)?; let cyclic_vk = buffer.read_target_verifier_circuit()?; - let use_keccak_tables = buffer.read_target_bool()?; + let mut table_in_use = Vec::with_capacity(NUM_TABLES); + for _ in 0..NUM_TABLES { + table_in_use.push(buffer.read_target_bool()?); + } Ok(Self { circuit, @@ -208,7 +213,7 @@ where index_verifier_data: index_verifier_data.try_into().unwrap(), public_values, cyclic_vk, - use_keccak_tables, + table_in_use: table_in_use.try_into().unwrap(), }) } } @@ -832,26 +837,24 @@ where let block_wrapper = Self::create_block_wrapper_circuit(&block); let two_to_one_block = Self::create_two_to_one_block_circuit(&block_wrapper); - // TODO(sdeng): enable more optional Tables let table_dummy_proofs = core::array::from_fn(|i| { - if KECCAK_TABLES_INDICES.contains(&i) { + if OPTIONAL_TABLE_INDICES.contains(&i) { let init_degree = degree_bits_ranges[i].start; - let common_circuit_data = by_table[i] + let chain = by_table[i] .by_stark_size .get(&init_degree) - .expect("Unable to get the shrinking circuits") + .expect("Unable to get the shrinking circuits"); + let common_circuit_data = chain .shrinking_wrappers .last() - .expect("Unable to get the last shrinking circuit") - .circuit - .common - .clone(); - let dummy_circuit: CircuitData = dummy_circuit(&common_circuit_data); + .map(|wrapper| &wrapper.circuit.common) + .unwrap_or(&chain.initial_wrapper.circuit.common); + let dummy_circuit: CircuitData = dummy_circuit(common_circuit_data); let dummy_pis = HashMap::new(); let proof = dummy_proof(&dummy_circuit, dummy_pis) .expect("Unable to generate dummy proofs"); Some(ShrunkProofData { - common_circuit_data, + common_circuit_data: common_circuit_data.clone(), proof, }) } else { @@ -900,8 +903,10 @@ where let mut builder = CircuitBuilder::new(CircuitConfig::standard_recursion_config()); - let use_keccak_tables = builder.add_virtual_bool_target_safe(); - let skip_keccak_tables = builder.not(use_keccak_tables); + let table_in_use: [BoolTarget; NUM_TABLES] = + core::array::from_fn(|_| builder.add_virtual_bool_target_safe()); + let table_not_in_use: [BoolTarget; NUM_TABLES] = + core::array::from_fn(|i| builder.not(table_in_use[i])); let public_values = add_virtual_public_values_public_input(&mut builder); let recursive_proofs = @@ -921,11 +926,17 @@ where } } + for (i, table) in table_in_use.iter().enumerate() { + if !OPTIONAL_TABLE_INDICES.contains(&i) { + builder.assert_one(table.target); + } + } + // Ensures that the trace cap is set to 0 when skipping Keccak tables. - for i in KECCAK_TABLES_INDICES { + for i in OPTIONAL_TABLE_INDICES { for h in &pis[i].trace_cap { for t in h { - let trace_cap_check = builder.mul(skip_keccak_tables.target, *t); + let trace_cap_check = builder.mul(table_not_in_use[i].target, *t); builder.assert_zero(trace_cap_check); } } @@ -941,16 +952,16 @@ where // Check that the correct CTL challenges are used in every proof. for (i, pi) in pis.iter().enumerate() { for j in 0..stark_config.num_challenges { - if KECCAK_TABLES_INDICES.contains(&i) { - // Ensures that the correct CTL challenges are used in Keccak tables when - // `enable_keccak_tables` is true. + if OPTIONAL_TABLE_INDICES.contains(&i) { + // Ensures that the correct CTL challenges are used when an optional table + // is in use. builder.conditional_assert_eq( - use_keccak_tables.target, + table_in_use[i].target, ctl_challenges.challenges[j].beta, pi.ctl_challenges.challenges[j].beta, ); builder.conditional_assert_eq( - use_keccak_tables.target, + table_in_use[i].target, ctl_challenges.challenges[j].gamma, pi.ctl_challenges.challenges[j].gamma, ); @@ -978,18 +989,18 @@ where let current_state_before = pis[i].challenger_state_before.as_ref(); let current_state_after = pis[i].challenger_state_after.as_ref(); for j in 0..state_len { - if KECCAK_TABLES_INDICES.contains(&i) { + if OPTIONAL_TABLE_INDICES.contains(&i) { // Ensure the challenger state: - // 1) prev == current_before when using Keccak + // 1) prev == current_before when using this table builder.conditional_assert_eq( - use_keccak_tables.target, + table_in_use[i].target, prev_state[j], current_state_before[j], ); - // 2) Update prev <- current_after when using Keccak - // 3) Keep prev <- prev when skipping Keccak + // 2) Update prev <- current_after when using this table + // 3) Keep prev <- prev when skipping this table prev_state[j] = - builder.select(use_keccak_tables, current_state_after[j], prev_state[j]); + builder.select(table_in_use[i], current_state_after[j], prev_state[j]); } else { builder.connect(prev_state[j], current_state_before[j]); prev_state[j] = current_state_after[j]; @@ -1017,11 +1028,10 @@ where .collect_vec(), ); - // Ensure that when Keccak tables are skipped, the Keccak tables' ctl_zs_first - // are all zeros. - for &i in KECCAK_TABLES_INDICES.iter() { + // Ensure that when a table is skipped, the table's ctl_zs_first are all zeros. + for &i in OPTIONAL_TABLE_INDICES.iter() { for &t in pis[i].ctl_zs_first.iter() { - let ctl_check = builder.mul(skip_keccak_tables.target, t); + let ctl_check = builder.mul(table_not_in_use[i].target, t); builder.assert_zero(ctl_check); } } @@ -1055,10 +1065,10 @@ where let inner_verifier_data = builder.random_access_verifier_data(index_verifier_data[i], possible_vks); - if KECCAK_TABLES_INDICES.contains(&i) { + if OPTIONAL_TABLE_INDICES.contains(&i) { builder .conditionally_verify_proof_or_dummy::( - use_keccak_tables, + table_in_use[i], &recursive_proofs[i], &inner_verifier_data, inner_common_data[i], @@ -1101,7 +1111,7 @@ where index_verifier_data, public_values, cyclic_vk, - use_keccak_tables, + table_in_use, } } @@ -2014,16 +2024,7 @@ where for table in 0..NUM_TABLES { let table_circuits = &self.by_table[table]; - if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { - let dummy_proof_data = self.table_dummy_proofs[table] - .as_ref() - .ok_or_else(|| anyhow::format_err!("No dummy_proof_data"))?; - root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO)?; - root_inputs.set_proof_with_pis_target( - &self.root.proof_with_pis[table], - &dummy_proof_data.proof, - )?; - } else { + if all_proof.table_in_use[table] { let stark_proof = &all_proof.multi_proof.stark_proofs[table] .as_ref() .ok_or_else(|| anyhow::format_err!("Unable to get stark proof"))?; @@ -2050,6 +2051,16 @@ where )?; root_inputs .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof)?; + } else { + assert!(OPTIONAL_TABLE_INDICES.contains(&table)); + let dummy_proof_data = self.table_dummy_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("No dummy_proof_data"))?; + root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO)?; + root_inputs.set_proof_with_pis_target( + &self.root.proof_with_pis[table], + &dummy_proof_data.proof, + )?; } check_abort_signal(abort_signal.clone())?; @@ -2069,7 +2080,11 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; - root_inputs.set_bool_target(self.root.use_keccak_tables, all_proof.use_keccak_tables)?; + self.root + .table_in_use + .iter() + .zip(all_proof.table_in_use.iter()) + .try_for_each(|(target, value)| root_inputs.set_bool_target(*target, *value))?; let root_proof = self.root.circuit.prove(root_inputs)?; @@ -2142,16 +2157,7 @@ where let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { - if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { - let dummy_proof = self.table_dummy_proofs[table] - .as_ref() - .ok_or_else(|| anyhow::format_err!("Unable to get dummpy proof"))?; - root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO)?; - root_inputs.set_proof_with_pis_target( - &self.root.proof_with_pis[table], - &dummy_proof.proof, - )?; - } else { + if all_proof.table_in_use[table] { let (table_circuit, index_verifier_data) = &table_circuits[table] .as_ref() .ok_or_else(|| anyhow::format_err!("Unable to get circuits"))?; @@ -2166,6 +2172,16 @@ where table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; root_inputs .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof)?; + } else { + assert!(OPTIONAL_TABLE_INDICES.contains(&table)); + let dummy_proof = self.table_dummy_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("Unable to get dummpy proof"))?; + root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO)?; + root_inputs.set_proof_with_pis_target( + &self.root.proof_with_pis[table], + &dummy_proof.proof, + )?; } check_abort_signal(abort_signal.clone())?; @@ -2185,7 +2201,11 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; - root_inputs.set_bool_target(self.root.use_keccak_tables, all_proof.use_keccak_tables)?; + self.root + .table_in_use + .iter() + .zip(all_proof.table_in_use.iter()) + .try_for_each(|(target, value)| root_inputs.set_bool_target(*target, *value))?; let root_proof = self.root.circuit.prove(root_inputs)?; @@ -3047,6 +3067,12 @@ where }); } + log::debug!( + "Table: {:?}, degree: {}, shrinking_wrappers_len: {}", + table, + degree_bits, + shrinking_wrappers.len() + ); Self { initial_wrapper, shrinking_wrappers, diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index a4994f686..db845f490 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -19,7 +19,8 @@ use GlobalMetadata::{ StateTrieRootDigestBefore, TransactionTrieRootDigestAfter, TransactionTrieRootDigestBefore, }; -use crate::all_stark::{AllStark, NUM_TABLES}; +use crate::all_stark::Table::MemAfter; +use crate::all_stark::{AllStark, Table, NUM_TABLES, OPTIONAL_TABLE_INDICES}; use crate::cpu::columns::CpuColumnsView; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; @@ -488,7 +489,7 @@ fn get_all_memory_address_and_values(memory_before: &MemoryState) -> Vec<(Memory pub struct TablesWithPVs { pub tables: [Vec>; NUM_TABLES], - pub use_keccak_tables: bool, + pub table_in_use: [bool; NUM_TABLES], pub public_values: PublicValues, } @@ -583,7 +584,28 @@ pub fn generate_traces, const D: usize>( mem_after: MemCap::default(), }; - let use_keccak_tables = !state.traces.keccak_inputs.is_empty(); + let mut table_in_use = [true; NUM_TABLES]; + if state.traces.keccak_inputs.is_empty() && OPTIONAL_TABLE_INDICES.contains(&Table::Keccak) { + assert!(OPTIONAL_TABLE_INDICES.contains(&Table::KeccakSponge)); + log::debug!("Keccak and KeccakSponge tables not in use"); + table_in_use[*Table::Keccak] = false; + table_in_use[*Table::KeccakSponge] = false; + } + if state.traces.logic_ops.is_empty() && OPTIONAL_TABLE_INDICES.contains(&Table::Logic) { + log::debug!("Logic table not in use"); + table_in_use[*Table::Logic] = false; + } + if state.traces.byte_packing_ops.is_empty() + && OPTIONAL_TABLE_INDICES.contains(&Table::BytePacking) + { + log::debug!("BytePacking table not in use"); + table_in_use[*Table::BytePacking] = false; + } + #[cfg(feature = "cdk_erigon")] + if state.traces.poseidon_ops.is_empty() && OPTIONAL_TABLE_INDICES.contains(&Table::Poseidon) { + log::debug!("Poseidon table not in use"); + table_in_use[*Table::Poseidon] = false; + } let tables = timed!( timing, @@ -598,9 +620,16 @@ pub fn generate_traces, const D: usize>( ) ); + let is_last_segment = + segment_data.registers_after.program_counter == KERNEL.global_labels["halt"]; + if is_last_segment && OPTIONAL_TABLE_INDICES.contains(&MemAfter) { + log::debug!("MemAfter table not in use"); + table_in_use[*MemAfter] = false; + } + Ok(TablesWithPVs { tables, - use_keccak_tables, + table_in_use, public_values, }) } diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index c7471471b..23a46d5c5 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -254,7 +254,7 @@ pub mod testing { use starky::lookup::{get_grand_product_challenge_set, GrandProductChallengeSet}; use starky::proof::StarkProofChallenges; - use crate::all_stark::KECCAK_TABLES_INDICES; + use crate::all_stark::OPTIONAL_TABLE_INDICES; use crate::get_challenges::observe_public_values; use crate::proof::*; use crate::witness::errors::ProgramError; @@ -282,7 +282,7 @@ pub mod testing { if let Some(stark_proof) = stark_proof { challenger.observe_cap(&stark_proof.proof.trace_cap); } else { - assert!(KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables); + assert!(OPTIONAL_TABLE_INDICES.contains(&i) && !self.table_in_use[i]); let zero_cap = vec![F::ZERO; config.fri_config.num_cap_elements() * NUM_HASH_OUT_ELTS]; challenger.observe_elements(&zero_cap); diff --git a/evm_arithmetization/src/proof.rs b/evm_arithmetization/src/proof.rs index b0de028af..f501a96d3 100644 --- a/evm_arithmetization/src/proof.rs +++ b/evm_arithmetization/src/proof.rs @@ -4,6 +4,7 @@ use ethereum_types::{Address, H256, U256}; use itertools::Itertools; use plonky2::field::extension::Extendable; use plonky2::hash::hash_types::{HashOutTarget, MerkleCapTarget, RichField, NUM_HASH_OUT_ELTS}; +use plonky2::hash::merkle_tree::MerkleCap; use plonky2::iop::target::{BoolTarget, Target}; use plonky2::plonk::circuit_builder::CircuitBuilder; use plonky2::plonk::config::{GenericConfig, GenericHashOut, Hasher}; @@ -47,10 +48,9 @@ pub struct AllProof, C: GenericConfig, co pub multi_proof: MultiProof, /// Public memory values used for the recursive proofs. pub public_values: PublicValues, - /// A flag indicating whether the Keccak and KeccakSponge tables contain - /// only padding values (i.e., no meaningful data). This is set to false - /// when no actual Keccak operations were performed. - pub use_keccak_tables: bool, + /// A flag indicating whether the table only contains padding values (i.e., + /// no meaningful data). + pub table_in_use: [bool; NUM_TABLES], } impl, C: GenericConfig, const D: usize> AllProof { @@ -603,6 +603,22 @@ impl MemCap { Self { mem_cap } } + + pub fn from_merkle_cap>(merkle_cap: MerkleCap) -> Self { + let mem_cap = merkle_cap + .0 + .iter() + .map(|h| { + h.to_vec() + .iter() + .map(|hi| hi.to_canonical_u64().into()) + .collect::>() + .try_into() + .unwrap() + }) + .collect::>(); + Self { mem_cap } + } } /// Memory values which are public. diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index 4192db6c8..698c23934 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -2,15 +2,15 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use anyhow::{anyhow, Result}; +use ethereum_types::U256; use itertools::Itertools; use once_cell::sync::Lazy; use plonky2::field::extension::Extendable; use plonky2::field::polynomial::PolynomialValues; use plonky2::fri::oracle::PolynomialBatch; use plonky2::hash::hash_types::RichField; -use plonky2::hash::merkle_tree::MerkleCap; use plonky2::iop::challenger::Challenger; -use plonky2::plonk::config::{GenericConfig, GenericHashOut}; +use plonky2::plonk::config::GenericConfig; use plonky2::timed; use plonky2::util::timing::TimingTree; use starky::config::StarkConfig; @@ -20,7 +20,7 @@ use starky::proof::StarkProofWithMetadata; use starky::prover::prove_with_commitment; use starky::stark::Stark; -use crate::all_stark::{AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES}; +use crate::all_stark::{AllStark, Table, NUM_TABLES, OPTIONAL_TABLE_INDICES}; use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::{generate_traces, GenerationInputs, TrimmedGenerationInputs}; use crate::get_challenges::observe_public_values; @@ -59,7 +59,7 @@ where all_stark, config, tables_with_pvs.tables, - tables_with_pvs.use_keccak_tables, + tables_with_pvs.table_in_use, &mut tables_with_pvs.public_values, timing, abort_signal, @@ -73,7 +73,7 @@ pub(crate) fn prove_with_traces( all_stark: &AllStark, config: &StarkConfig, trace_poly_values: [Vec>; NUM_TABLES], - use_keccak_tables: bool, + table_in_use: [bool; NUM_TABLES], public_values: &mut PublicValues, timing: &mut TimingTree, abort_signal: Option>, @@ -117,7 +117,7 @@ where .collect::>(); let mut challenger = Challenger::::new(); for (i, cap) in trace_caps.iter().enumerate() { - if KECCAK_TABLES_INDICES.contains(&i) && !use_keccak_tables { + if OPTIONAL_TABLE_INDICES.contains(&i) && !table_in_use[i] { // Observe zero merkle caps when skipping Keccak tables. let zero_merkle_cap = cap.flatten().iter().map(|_| F::ZERO).collect::>(); challenger.observe_elements(&zero_merkle_cap); @@ -151,7 +151,7 @@ where config, &trace_poly_values, trace_commitments, - use_keccak_tables, + table_in_use, ctl_data_per_table, &mut challenger, &ctl_challenges, @@ -159,34 +159,8 @@ where abort_signal, )? ); - public_values.mem_before = MemCap { - mem_cap: mem_before_cap - .0 - .iter() - .map(|h| { - h.to_vec() - .iter() - .map(|hi| hi.to_canonical_u64().into()) - .collect::>() - .try_into() - .unwrap() - }) - .collect::>(), - }; - public_values.mem_after = MemCap { - mem_cap: mem_after_cap - .0 - .iter() - .map(|h| { - h.to_vec() - .iter() - .map(|hi| hi.to_canonical_u64().into()) - .collect::>() - .try_into() - .unwrap() - }) - .collect::>(), - }; + public_values.mem_before = mem_before_cap; + public_values.mem_after = mem_after_cap; // This is an expensive check, hence is only run when `debug_assertions` are // enabled. @@ -215,14 +189,14 @@ where ctl_challenges, }, public_values: public_values.clone(), - use_keccak_tables, + table_in_use, }) } -type ProofWithMemCaps = ( +type ProofWithMemCaps = ( [Option>; NUM_TABLES], - MerkleCap, - MerkleCap, + MemCap, + MemCap, ); /// Generates a proof for each STARK. @@ -239,80 +213,85 @@ fn prove_with_commitments( config: &StarkConfig, trace_poly_values: &[Vec>; NUM_TABLES], trace_commitments: Vec>, - use_keccak_tables: bool, + table_in_use: [bool; NUM_TABLES], ctl_data_per_table: [CtlData; NUM_TABLES], challenger: &mut Challenger, ctl_challenges: &GrandProductChallengeSet, timing: &mut TimingTree, abort_signal: Option>, -) -> Result> +) -> Result> where F: RichField + Extendable, C: GenericConfig, { macro_rules! prove_table { ($stark:ident, $table:expr) => { - timed!( - timing, - &format!("prove {} STARK", stringify!($stark)), - prove_single_table( - &all_stark.$stark, - config, - &trace_poly_values[*$table], - &trace_commitments[*$table], - &ctl_data_per_table[*$table], - ctl_challenges, - challenger, + if table_in_use[*$table] { + Some(timed!( timing, - abort_signal.clone(), - )? - ) + &format!("prove {} STARK", stringify!($stark)), + prove_single_table( + &all_stark.$stark, + config, + &trace_poly_values[*$table], + &trace_commitments[*$table], + &ctl_data_per_table[*$table], + ctl_challenges, + challenger, + timing, + abort_signal.clone(), + )? + )) + } else { + None + } }; } - let (arithmetic_proof, _) = prove_table!(arithmetic_stark, Table::Arithmetic); - let (byte_packing_proof, _) = prove_table!(byte_packing_stark, Table::BytePacking); - let (cpu_proof, _) = prove_table!(cpu_stark, Table::Cpu); - let keccak_proof = if use_keccak_tables { - Some(prove_table!(keccak_stark, Table::Keccak).0) - } else { - None - }; - let keccak_sponge_proof = if use_keccak_tables { - Some(prove_table!(keccak_sponge_stark, Table::KeccakSponge).0) - } else { - None - }; - let (logic_proof, _) = prove_table!(logic_stark, Table::Logic); - let (memory_proof, _) = prove_table!(memory_stark, Table::Memory); - let (mem_before_proof, mem_before_cap) = prove_table!(mem_before_stark, Table::MemBefore); - let (mem_after_proof, mem_after_cap) = prove_table!(mem_after_stark, Table::MemAfter); + let arithmetic_proof = prove_table!(arithmetic_stark, Table::Arithmetic); + let byte_packing_proof = prove_table!(byte_packing_stark, Table::BytePacking); + let cpu_proof = prove_table!(cpu_stark, Table::Cpu); + let keccak_proof = prove_table!(keccak_stark, Table::Keccak); + let keccak_sponge_proof = prove_table!(keccak_sponge_stark, Table::KeccakSponge); + let logic_proof = prove_table!(logic_stark, Table::Logic); + let memory_proof = prove_table!(memory_stark, Table::Memory); + let mem_before_proof = prove_table!(mem_before_stark, Table::MemBefore); + let mem_after_proof = prove_table!(mem_after_stark, Table::MemAfter); + + let mem_before_cap = trace_commitments[*Table::MemBefore].merkle_tree.cap.clone(); + let mem_before_cap = MemCap::from_merkle_cap(mem_before_cap); + let mem_after_cap = trace_commitments[*Table::MemAfter].merkle_tree.cap.clone(); + let mut mem_after_cap = MemCap::from_merkle_cap(mem_after_cap); + if !table_in_use[*Table::MemAfter] { + for hash in &mut mem_after_cap.mem_cap { + for element in hash.iter_mut() { + *element = U256::zero(); + } + } + } #[cfg(feature = "cdk_erigon")] - let (poseidon_proof, _) = prove_table!(poseidon_stark, Table::Poseidon); + let poseidon_proof = prove_table!(poseidon_stark, Table::Poseidon); Ok(( [ - Some(arithmetic_proof), - Some(byte_packing_proof), - Some(cpu_proof), + arithmetic_proof, + byte_packing_proof, + cpu_proof, keccak_proof, keccak_sponge_proof, - Some(logic_proof), - Some(memory_proof), - Some(mem_before_proof), - Some(mem_after_proof), + logic_proof, + memory_proof, + mem_before_proof, + mem_after_proof, #[cfg(feature = "cdk_erigon")] - Some(poseidon_proof), + poseidon_proof, ], mem_before_cap, mem_after_cap, )) } -type ProofSingleWithCap = - (StarkProofWithMetadata, MerkleCap); - /// Computes a proof for a single STARK table, including: /// - the initial state of the challenger, /// - all the requires Merkle caps, @@ -329,7 +308,7 @@ pub(crate) fn prove_single_table( challenger: &mut Challenger, timing: &mut TimingTree, abort_signal: Option>, -) -> Result> +) -> Result> where F: RichField + Extendable, C: GenericConfig, @@ -356,7 +335,7 @@ where init_challenger_state, })?; - Ok((proof, trace_commitment.merkle_tree.cap.clone())) + Ok(proof) } /// Utility method that checks whether a kill signal has been emitted by one of diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 039dc2504..7e10801f7 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -18,7 +18,7 @@ use crate::generation::{TrieInputs, TrimmedGenerationInputs}; use crate::proof::TrieRoots; use crate::witness::operation::Operation; use crate::{ - generation::mpt::AccountRlp, proof::BlockMetadata, util::h2u, GenerationInputs, + generation::mpt::AccountRlp, logic, proof::BlockMetadata, util::h2u, GenerationInputs, GenerationSegmentData, SegmentDataIterator, }; @@ -229,8 +229,12 @@ pub fn segment_with_empty_tables() -> Result<( SegmentDataIterator::::new(&payload, max_cpu_len_log); let (trimmed_inputs, segment_data) = segment_iterator.next().unwrap()?; + // Ensures that there are no Keccak and Logic ops in the segment. let opcode_counts = &segment_data.opcode_counts; assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); + assert!(!opcode_counts.contains_key(&Operation::BinaryLogic(logic::Op::And))); + assert!(!opcode_counts.contains_key(&Operation::BinaryLogic(logic::Op::Or))); + assert!(!opcode_counts.contains_key(&Operation::BinaryLogic(logic::Op::Xor))); Ok((trimmed_inputs, segment_data)) } diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index e7b0c5198..1d4fdf387 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -93,7 +93,7 @@ pub mod testing { use starky::stark::Stark; use starky::verifier::verify_stark_proof_with_challenges; - use crate::all_stark::{Table, MEMORY_CTL_IDX, NUM_CTLS}; + use crate::all_stark::{Table, MEMORY_CTL_IDX, NUM_CTLS, OPTIONAL_TABLE_INDICES}; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; use crate::get_challenges::testing::AllProofChallenges; @@ -124,6 +124,22 @@ pub mod testing { self.poseidon_stark.num_lookup_helper_columns(config), ] } + + pub fn get_constraint_degree(&self, table: Table) -> usize { + match table { + Table::Arithmetic => self.arithmetic_stark.constraint_degree(), + Table::BytePacking => self.byte_packing_stark.constraint_degree(), + Table::Cpu => self.cpu_stark.constraint_degree(), + Table::Keccak => self.keccak_stark.constraint_degree(), + Table::KeccakSponge => self.keccak_sponge_stark.constraint_degree(), + Table::Logic => self.logic_stark.constraint_degree(), + Table::Memory => self.memory_stark.constraint_degree(), + Table::MemBefore => self.mem_before_stark.constraint_degree(), + Table::MemAfter => self.mem_after_stark.constraint_degree(), + #[cfg(feature = "cdk_erigon")] + Table::Poseidon => self.poseidon_stark.constraint_degree(), + } + } } fn verify_initial_memory< @@ -191,48 +207,48 @@ pub mod testing { macro_rules! verify_table { ($stark:ident, $table:expr) => { - let stark_proof = &stark_proofs[*$table] - .as_ref() - .expect("Missing stark_proof") - .proof; - let ctl_vars = { - let (total_num_helpers, _, num_helpers_by_ctl) = - CrossTableLookup::num_ctl_helpers_zs_all( - &all_stark.cross_table_lookups, - *$table, - config.num_challenges, - $stark.constraint_degree(), - ); - CtlCheckVars::from_proof( - *$table, - &stark_proof, - &all_stark.cross_table_lookups, - &ctl_challenges, - num_lookup_columns[*$table], - total_num_helpers, - &num_helpers_by_ctl, - ) - }; - verify_stark_proof_with_challenges( - $stark, - stark_proof, - &stark_challenges[*$table] + if !OPTIONAL_TABLE_INDICES.contains(&*$table) || all_proof.table_in_use[*$table] { + let stark_proof = &stark_proofs[*$table] .as_ref() - .expect("Missing challenges"), - Some(&ctl_vars), - &[], - config, - )?; + .expect("Missing stark_proof") + .proof; + let ctl_vars = { + let (total_num_helpers, _, num_helpers_by_ctl) = + CrossTableLookup::num_ctl_helpers_zs_all( + &all_stark.cross_table_lookups, + *$table, + config.num_challenges, + $stark.constraint_degree(), + ); + CtlCheckVars::from_proof( + *$table, + &stark_proof, + &all_stark.cross_table_lookups, + &ctl_challenges, + num_lookup_columns[*$table], + total_num_helpers, + &num_helpers_by_ctl, + ) + }; + verify_stark_proof_with_challenges( + $stark, + stark_proof, + &stark_challenges[*$table] + .as_ref() + .expect("Missing challenges"), + Some(&ctl_vars), + &[], + config, + )?; + } }; } verify_table!(arithmetic_stark, Table::Arithmetic); verify_table!(byte_packing_stark, Table::BytePacking); verify_table!(cpu_stark, Table::Cpu); - if all_proof.use_keccak_tables { - verify_table!(keccak_stark, Table::Keccak); - verify_table!(keccak_sponge_stark, Table::KeccakSponge); - } + verify_table!(keccak_stark, Table::Keccak); + verify_table!(keccak_sponge_stark, Table::KeccakSponge); verify_table!(logic_stark, Table::Logic); verify_table!(memory_stark, Table::Memory); verify_table!(mem_before_stark, Table::MemBefore); @@ -263,37 +279,34 @@ pub mod testing { let all_ctls = &all_stark.cross_table_lookups; + let table_all = Table::all(); + let ctl_zs_first_values = core::array::from_fn(|i| { + let table = table_all[i]; + if let Some(stark_proof) = &stark_proofs[i] { + stark_proof + .proof + .openings + .ctl_zs_first + .as_ref() + .expect("Missing ctl_zs") + .clone() + } else if OPTIONAL_TABLE_INDICES.contains(&table) { + let degree = all_stark.get_constraint_degree(table); + let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( + all_ctls, + i, + config.num_challenges, + degree, + ); + vec![F::ZERO; n] + } else { + panic!("Unable to find stark_proof for table {:?}", table); + } + }); + verify_cross_table_lookups::( cross_table_lookups, - core::array::from_fn(|i| { - if let Some(stark_proof) = &stark_proofs[i] { - stark_proof - .proof - .openings - .ctl_zs_first - .as_ref() - .expect("Missing ctl_zs") - .clone() - } else if i == *Table::Keccak { - let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( - all_ctls, - *Table::Keccak, - config.num_challenges, - keccak_stark.constraint_degree(), - ); - vec![F::ZERO; n] - } else if i == *Table::KeccakSponge { - let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( - all_ctls, - *Table::KeccakSponge, - config.num_challenges, - keccak_sponge_stark.constraint_degree(), - ); - vec![F::ZERO; n] - } else { - panic!("Unable to find stark_proof"); - } - }), + ctl_zs_first_values, &extra_looking_sums, config, ) From f8c5f9cdab1dd2bbf7e54cd23a597e1c1b2db3e8 Mon Sep 17 00:00:00 2001 From: Sai <135601871+sai-deng@users.noreply.github.com> Date: Sun, 20 Oct 2024 00:03:12 +0000 Subject: [PATCH 07/17] fix (#738) --- evm_arithmetization/src/generation/mod.rs | 6 ++--- evm_arithmetization/src/witness/traces.rs | 31 +++++++++++++---------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index db845f490..6ec02ea73 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -607,7 +607,7 @@ pub fn generate_traces, const D: usize>( table_in_use[*Table::Poseidon] = false; } - let tables = timed!( + let (tables, final_len) = timed!( timing, "convert trace data to tables", state.traces.into_tables( @@ -620,9 +620,7 @@ pub fn generate_traces, const D: usize>( ) ); - let is_last_segment = - segment_data.registers_after.program_counter == KERNEL.global_labels["halt"]; - if is_last_segment && OPTIONAL_TABLE_INDICES.contains(&MemAfter) { + if final_len == 0 && OPTIONAL_TABLE_INDICES.contains(&MemAfter) { log::debug!("MemAfter table not in use"); table_in_use[*MemAfter] = false; } diff --git a/evm_arithmetization/src/witness/traces.rs b/evm_arithmetization/src/witness/traces.rs index cb9e605e8..e1daa53b0 100644 --- a/evm_arithmetization/src/witness/traces.rs +++ b/evm_arithmetization/src/witness/traces.rs @@ -144,7 +144,7 @@ impl Traces { mut trace_lengths: TraceCheckpoint, config: &StarkConfig, timing: &mut TimingTree, - ) -> [Vec>; NUM_TABLES] + ) -> ([Vec>; NUM_TABLES], usize) where T: RichField + Extendable, { @@ -240,19 +240,22 @@ impl Traces { final_values.len() ); - [ - arithmetic_trace, - byte_packing_trace, - cpu_trace, - keccak_trace, - keccak_sponge_trace, - logic_trace, - memory_trace, - mem_before_trace, - mem_after_trace, - #[cfg(feature = "cdk_erigon")] - poseidon_trace, - ] + ( + [ + arithmetic_trace, + byte_packing_trace, + cpu_trace, + keccak_trace, + keccak_sponge_trace, + logic_trace, + memory_trace, + mem_before_trace, + mem_after_trace, + #[cfg(feature = "cdk_erigon")] + poseidon_trace, + ], + final_values.len(), + ) } } From 8681cd713121a7c7ea7607d39e2a98502e8e144b Mon Sep 17 00:00:00 2001 From: 0xaatif <169152398+0xaatif@users.noreply.github.com> Date: Mon, 21 Oct 2024 15:45:00 +0100 Subject: [PATCH 08/17] fix: more robust SMT parsing (#733) * mark: 0xaatif/smt-parse-tweaking * robustness --- trace_decoder/src/type2.rs | 3 +++ trace_decoder/src/wire.rs | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/trace_decoder/src/type2.rs b/trace_decoder/src/type2.rs index a71761533..44d13e89a 100644 --- a/trace_decoder/src/type2.rs +++ b/trace_decoder/src/type2.rs @@ -173,8 +173,10 @@ fn visit( address, value, }) => { + ensure!(address.len() == Address::len_bytes()); let address = Address::from_slice(&address); let collated = leaves.entry(address).or_default(); + ensure!(value.len() <= 32); let value = U256::from_big_endian(&value); macro_rules! ensure { ($expr:expr) => { @@ -195,6 +197,7 @@ fn visit( collated.code = Some(value) } SmtLeafType::Storage(slot) => { + ensure!(slot.len() <= 32); let clobbered = collated.storage.insert(U256::from_big_endian(&slot), value); ensure!(clobbered.is_none()) } diff --git a/trace_decoder/src/wire.rs b/trace_decoder/src/wire.rs index 63dee6040..359043057 100644 --- a/trace_decoder/src/wire.rs +++ b/trace_decoder/src/wire.rs @@ -80,8 +80,6 @@ pub enum Instruction { } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/705 -// `address` and `value` should be fixed length fields pub struct SmtLeaf { pub node_type: SmtLeafType, pub address: NonEmpty>, @@ -89,8 +87,6 @@ pub struct SmtLeaf { } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/705 -// `Storage` should contain a fixed length field pub enum SmtLeafType { Balance, Nonce, From e2b82e7ec2af9687b7b56632ed2923566bcb94f6 Mon Sep 17 00:00:00 2001 From: Sai <135601871+sai-deng@users.noreply.github.com> Date: Mon, 21 Oct 2024 15:33:54 +0000 Subject: [PATCH 09/17] Move opcode_count Under Test Configuration (#736) * opcode count behind cfg test * add unit test * fix warning * cargo fmt --- .../src/cpu/kernel/interpreter.rs | 16 ++++++++--- .../src/generation/segments.rs | 20 ++++++++++--- evm_arithmetization/src/testing_utils.rs | 28 ++++++++++++++----- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 94a3d20fb..d9745504e 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -54,8 +54,6 @@ pub(crate) struct Interpreter { /// The interpreter will halt only if the current context matches /// halt_context pub(crate) halt_context: Option, - /// Counts the number of appearances of each opcode. For debugging purposes. - pub(crate) opcode_count: HashMap, jumpdest_table: HashMap>, /// `true` if the we are currently carrying out a jumpdest analysis. pub(crate) is_jumpdest_analysis: bool, @@ -64,6 +62,10 @@ pub(crate) struct Interpreter { pub(crate) clock: usize, /// Log of the maximal number of CPU cycles in one segment execution. max_cpu_len_log: Option, + + #[cfg(test)] + // Counts the number of appearances of each opcode. For debugging purposes. + pub(crate) opcode_count: HashMap, } /// Simulates the CPU execution from `state` until the program counter reaches @@ -178,6 +180,7 @@ impl Interpreter { // while the label `halt` is the halting label in the kernel. halt_offsets: vec![DEFAULT_HALT_OFFSET, KERNEL.global_labels["halt_final"]], halt_context: None, + #[cfg(test)] opcode_count: HashMap::new(), jumpdest_table: HashMap::new(), is_jumpdest_analysis: false, @@ -209,6 +212,7 @@ impl Interpreter { generation_state: state.soft_clone(), halt_offsets: vec![halt_offset], halt_context: Some(halt_context), + #[cfg(test)] opcode_count: HashMap::new(), jumpdest_table: HashMap::new(), is_jumpdest_analysis: true, @@ -428,6 +432,7 @@ impl Interpreter { self.max_cpu_len_log } + #[cfg(test)] pub(crate) fn reset_opcode_counts(&mut self) { self.opcode_count = HashMap::new(); } @@ -669,8 +674,11 @@ impl State for Interpreter { let op = decode(registers, opcode)?; - // Increment the opcode count - *self.opcode_count.entry(op).or_insert(0) += 1; + #[cfg(test)] + { + // Increment the opcode count + *self.opcode_count.entry(op).or_insert(0) += 1; + } fill_op_flag(op, &mut row); diff --git a/evm_arithmetization/src/generation/segments.rs b/evm_arithmetization/src/generation/segments.rs index 36bac8f8a..1df63af29 100644 --- a/evm_arithmetization/src/generation/segments.rs +++ b/evm_arithmetization/src/generation/segments.rs @@ -1,6 +1,7 @@ //! Module defining the logic around proof segmentation into chunks, //! which allows what is commonly known as zk-continuations. +#[cfg(test)] use std::collections::HashMap; use anyhow::Result; @@ -13,6 +14,7 @@ use crate::cpu::kernel::interpreter::{set_registers_and_run, ExtraSegmentData, I use crate::generation::state::State; use crate::generation::{collect_debug_tries, debug_inputs, ErrorWithTries, GenerationInputs}; use crate::witness::memory::MemoryState; +#[cfg(test)] use crate::witness::operation::Operation; use crate::witness::state::RegistersState; @@ -32,7 +34,9 @@ pub struct GenerationSegmentData { pub(crate) extra_data: ExtraSegmentData, /// Log of the maximal cpu length. pub(crate) max_cpu_len_log: Option, - /// Counts the number of appearances of each opcode. For debugging purposes. + + #[cfg(test)] + // Counts the number of appearances of each opcode. For debugging purposes. pub(crate) opcode_counts: HashMap, } @@ -82,6 +86,7 @@ fn build_segment_data( access_lists_ptrs: interpreter.generation_state.access_lists_ptrs.clone(), state_ptrs: interpreter.generation_state.state_ptrs.clone(), }, + #[cfg(test)] opcode_counts: interpreter.opcode_count.clone(), } } @@ -139,8 +144,11 @@ impl SegmentDataIterator { let segment_index = segment_data.segment_index; - // Reset opcode counts before executing the segment - self.interpreter.reset_opcode_counts(); + #[cfg(test)] + { + // Reset opcode counts before executing the segment + self.interpreter.reset_opcode_counts(); + } // Run the interpreter to get `registers_after` and the partial data for the // next segment. @@ -156,7 +164,11 @@ impl SegmentDataIterator { )); segment_data.registers_after = updated_registers; - segment_data.opcode_counts = self.interpreter.opcode_count.clone(); + + #[cfg(test)] + { + segment_data.opcode_counts = self.interpreter.opcode_count.clone(); + } Ok(Some(Box::new((segment_data, partial_segment_data)))) } else { diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 7e10801f7..321e7fe07 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -16,9 +16,10 @@ pub use crate::cpu::kernel::cancun_constants::*; pub use crate::cpu::kernel::constants::global_exit_root::*; use crate::generation::{TrieInputs, TrimmedGenerationInputs}; use crate::proof::TrieRoots; +#[cfg(test)] use crate::witness::operation::Operation; use crate::{ - generation::mpt::AccountRlp, logic, proof::BlockMetadata, util::h2u, GenerationInputs, + generation::mpt::AccountRlp, proof::BlockMetadata, util::h2u, GenerationInputs, GenerationSegmentData, SegmentDataIterator, }; @@ -229,12 +230,25 @@ pub fn segment_with_empty_tables() -> Result<( SegmentDataIterator::::new(&payload, max_cpu_len_log); let (trimmed_inputs, segment_data) = segment_iterator.next().unwrap()?; + Ok((trimmed_inputs, segment_data)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::logic; + // Ensures that there are no Keccak and Logic ops in the segment. - let opcode_counts = &segment_data.opcode_counts; - assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); - assert!(!opcode_counts.contains_key(&Operation::BinaryLogic(logic::Op::And))); - assert!(!opcode_counts.contains_key(&Operation::BinaryLogic(logic::Op::Or))); - assert!(!opcode_counts.contains_key(&Operation::BinaryLogic(logic::Op::Xor))); + #[test] + fn test_segment_with_empty_tables() -> Result<()> { + let (_, segment_data) = segment_with_empty_tables()?; - Ok((trimmed_inputs, segment_data)) + let opcode_counts = &segment_data.opcode_counts; + assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); + assert!(!opcode_counts.contains_key(&Operation::BinaryLogic(logic::Op::And))); + assert!(!opcode_counts.contains_key(&Operation::BinaryLogic(logic::Op::Or))); + assert!(!opcode_counts.contains_key(&Operation::BinaryLogic(logic::Op::Xor))); + + Ok(()) + } } From b1b73e32b41bce1b81d15d64196036358a68642c Mon Sep 17 00:00:00 2001 From: 0xaatif <169152398+0xaatif@users.noreply.github.com> Date: Tue, 22 Oct 2024 14:04:18 +0100 Subject: [PATCH 10/17] feat: trait World for `trace_decoder` (#732) --- Cargo.lock | 10 - trace_decoder/Cargo.toml | 1 - trace_decoder/src/core.rs | 431 +++++++++++++--------------------- trace_decoder/src/lib.rs | 16 +- trace_decoder/src/observer.rs | 23 +- trace_decoder/src/tries.rs | 375 +++++------------------------ trace_decoder/src/type1.rs | 23 +- trace_decoder/src/type2.rs | 128 +--------- trace_decoder/src/world.rs | 420 +++++++++++++++++++++++++++++++++ zero/src/bin/trie_diff.rs | 4 +- 10 files changed, 680 insertions(+), 751 deletions(-) create mode 100644 trace_decoder/src/world.rs diff --git a/Cargo.lock b/Cargo.lock index ef891b1e1..06c4aa6a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1151,15 +1151,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "build-array" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ef4e2687af237b2646687e19a0643bc369878216122e46c3f1a01c56baa9d5" -dependencies = [ - "arrayvec", -] - [[package]] name = "bumpalo" version = "3.16.0" @@ -5136,7 +5127,6 @@ dependencies = [ "assert2", "bitflags 2.6.0", "bitvec", - "build-array", "bytes", "camino", "ciborium", diff --git a/trace_decoder/Cargo.toml b/trace_decoder/Cargo.toml index 5bdd24f12..4b8d8e7bc 100644 --- a/trace_decoder/Cargo.toml +++ b/trace_decoder/Cargo.toml @@ -15,7 +15,6 @@ alloy-compat = "0.1.0" anyhow.workspace = true bitflags.workspace = true bitvec.workspace = true -build-array = "0.1.2" bytes.workspace = true ciborium.workspace = true ciborium-io.workspace = true diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 46495030f..8093098c1 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -4,12 +4,11 @@ use std::{ mem, }; -use alloy_compat::Compat as _; use anyhow::{anyhow, bail, ensure, Context as _}; use either::Either; -use ethereum_types::{Address, U256}; +use ethereum_types::{Address, BigEndianHash as _, U256}; use evm_arithmetization::{ - generation::{mpt::AccountRlp, TrieInputs}, + generation::TrieInputs, proof::{BlockMetadata, TrieRoots}, GenerationInputs, }; @@ -21,10 +20,11 @@ use zk_evm_common::gwei_to_wei; use crate::{ observer::{DummyObserver, Observer}, - tries::StateSmt, + world::Type2World, }; use crate::{ - tries::{MptKey, ReceiptTrie, StateMpt, StateTrie, StorageTrie, TransactionTrie}, + tries::{MptKey, ReceiptTrie, StateMpt, StorageTrie, TransactionTrie}, + world::{Type1World, World}, BlockLevelData, BlockTrace, BlockTraceTriePreImages, CombinedPreImages, ContractCodeUsage, OtherBlockData, SeparateStorageTriesPreImage, SeparateTriePreImage, SeparateTriePreImages, TxnInfo, TxnMeta, TxnTrace, @@ -46,7 +46,7 @@ pub fn entrypoint( trace: BlockTrace, other: OtherBlockData, batch_size_hint: usize, - observer: &mut impl Observer, + observer: &mut impl Observer, wire_disposition: WireDisposition, ) -> anyhow::Result> { ensure!(batch_size_hint != 0); @@ -61,7 +61,7 @@ pub fn entrypoint( BlockTraceTriePreImages::Separate(_) => FatalMissingCode(true), BlockTraceTriePreImages::Combined(_) => FatalMissingCode(false), }; - let (state, storage, mut code) = start(trie_pre_images, wire_disposition)?; + let (world, mut code) = start(trie_pre_images, wire_disposition)?; code.extend(code_db); @@ -82,11 +82,10 @@ pub fn entrypoint( *amt = gwei_to_wei(*amt) } - let batches = match state { - Either::Left(mpt) => Either::Left( + let batches = match world { + Either::Left(type1world) => Either::Left( middle( - mpt, - storage, + type1world, batch(txn_info, batch_size_hint), &mut code, &b_meta, @@ -98,11 +97,10 @@ pub fn entrypoint( .into_iter() .map(|it| it.map(Either::Left)), ), - Either::Right(smt) => { + Either::Right(type2world) => { Either::Right( middle( - smt, - storage, + type2world, batch(txn_info, batch_size_hint), &mut code, &b_meta, @@ -128,42 +126,43 @@ pub fn entrypoint( byte_code, before: IntraBlockTries { - state, - storage, + world, transaction, receipt, }, after, withdrawals, - }| GenerationInputs { - txn_number_before: first_txn_ix.into(), - gas_used_before: running_gas_used.into(), - gas_used_after: { - running_gas_used += gas_used; - running_gas_used.into() - }, - signed_txns: byte_code.into_iter().map(Into::into).collect(), - withdrawals, - ger_data, - tries: TrieInputs { - state_trie: match state { - Either::Left(mpt) => mpt.into(), - Either::Right(_) => todo!("evm_arithmetization accepts an SMT"), + }| { + let (state, storage) = world + .expect_left("TODO(0xaatif): evm_arithemetization accepts an SMT") + .into_state_and_storage(); + GenerationInputs { + txn_number_before: first_txn_ix.into(), + gas_used_before: running_gas_used.into(), + gas_used_after: { + running_gas_used += gas_used; + running_gas_used.into() }, - transactions_trie: transaction.into(), - receipts_trie: receipt.into(), - storage_tries: storage.into_iter().map(|(k, v)| (k, v.into())).collect(), - }, - trie_roots_after: after, - checkpoint_state_trie_root, - checkpoint_consolidated_hash, - contract_code: contract_code - .into_iter() - .map(|it| (keccak_hash::keccak(&it), it)) - .collect(), - block_metadata: b_meta.clone(), - block_hashes: b_hashes.clone(), - burn_addr, + signed_txns: byte_code.into_iter().map(Into::into).collect(), + withdrawals, + ger_data, + tries: TrieInputs { + state_trie: state.into(), + transactions_trie: transaction.into(), + receipts_trie: receipt.into(), + storage_tries: storage.into_iter().map(|(k, v)| (k, v.into())).collect(), + }, + trie_roots_after: after, + checkpoint_state_trie_root, + checkpoint_consolidated_hash, + contract_code: contract_code + .into_iter() + .map(|it| (keccak_hash::keccak(&it), it)) + .collect(), + block_metadata: b_meta.clone(), + block_hashes: b_hashes.clone(), + burn_addr, + } }, ) .collect()) @@ -178,11 +177,7 @@ pub fn entrypoint( fn start( pre_images: BlockTraceTriePreImages, wire_disposition: WireDisposition, -) -> anyhow::Result<( - Either, - BTreeMap, - Hash2Code, -)> { +) -> anyhow::Result<(Either, Hash2Code)> { Ok(match pre_images { // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/401 // refactor our convoluted input types @@ -190,27 +185,26 @@ fn start( state: SeparateTriePreImage::Direct(state), storage: SeparateStorageTriesPreImage::MultipleTries(storage), }) => { - let state = state.items().try_fold( - StateMpt::default(), - |mut acc, (nibbles, hash_or_val)| { - let path = MptKey::from_nibbles(nibbles); - match hash_or_val { - mpt_trie::trie_ops::ValOrHash::Val(bytes) => { - #[expect(deprecated)] // this is MPT specific - acc.insert_by_hashed_address( - path.into_hash() - .context("invalid path length in direct state trie")?, - rlp::decode(&bytes) - .context("invalid AccountRlp in direct state trie")?, - )?; - } - mpt_trie::trie_ops::ValOrHash::Hash(h) => { - acc.insert_hash_by_key(path, h)?; - } - }; - anyhow::Ok(acc) - }, - )?; + let state = + state + .items() + .try_fold(StateMpt::new(), |mut acc, (nibbles, hash_or_val)| { + let path = MptKey::from_nibbles(nibbles); + match hash_or_val { + mpt_trie::trie_ops::ValOrHash::Val(bytes) => { + acc.insert( + path.into_hash() + .context("invalid path length in direct state trie")?, + rlp::decode(&bytes) + .context("invalid AccountRlp in direct state trie")?, + )?; + } + mpt_trie::trie_ops::ValOrHash::Hash(h) => { + acc.insert_hash(path, h)?; + } + }; + anyhow::Ok(acc) + })?; let storage = storage .into_iter() .map(|(k, SeparateTriePreImage::Direct(v))| { @@ -230,12 +224,15 @@ fn start( .map(|v| (k, v)) }) .collect::>()?; - (Either::Left(state), storage, Hash2Code::new()) + ( + Either::Left(Type1World::new(state, storage)?), + Hash2Code::new(), + ) } BlockTraceTriePreImages::Combined(CombinedPreImages { compact }) => { let instructions = crate::wire::parse(&compact) .context("couldn't parse instructions from binary format")?; - let (state, storage, code) = match wire_disposition { + match wire_disposition { WireDisposition::Type1 => { let crate::type1::Frontend { state, @@ -243,22 +240,19 @@ fn start( code, } = crate::type1::frontend(instructions)?; ( - Either::Left(state), - storage, + Either::Left(Type1World::new(state, storage)?), Hash2Code::from_iter(code.into_iter().map(NonEmpty::into_vec)), ) } WireDisposition::Type2 => { - let crate::type2::Frontend { trie, code } = + let crate::type2::Frontend { world: trie, code } = crate::type2::frontend(instructions)?; ( Either::Right(trie), - BTreeMap::new(), Hash2Code::from_iter(code.into_iter().map(NonEmpty::into_vec)), ) } - }; - (state, storage, code) + } } }) } @@ -358,24 +352,21 @@ impl Batch { /// [`evm_arithmetization::generation::TrieInputs`], /// generic over state trie representation. #[derive(Debug)] -pub struct IntraBlockTries { - pub state: StateTrieT, - pub storage: BTreeMap, +pub struct IntraBlockTries { + pub world: WorldT, pub transaction: TransactionTrie, pub receipt: ReceiptTrie, } impl IntraBlockTries { - fn map(self, mut f: impl FnMut(T) -> U) -> IntraBlockTries { + fn map(self, f: impl FnOnce(T) -> U) -> IntraBlockTries { let Self { - state, - storage, + world, transaction, receipt, } = self; IntraBlockTries { - state: f(state), - storage, + world: f(world), transaction, receipt, } @@ -391,11 +382,9 @@ pub struct FatalMissingCode(pub bool); /// Does the main work mentioned in the [module documentation](super). #[allow(clippy::too_many_arguments)] -fn middle( +fn middle( // state at the beginning of the block - mut state_trie: StateTrieT, - // storage at the beginning of the block - mut storage_tries: BTreeMap, + mut world: WorldT, // None represents a dummy transaction that should not increment the transaction index // all batches SHOULD not be empty batches: Vec>>, @@ -406,25 +395,11 @@ fn middle( mut withdrawals: Vec<(Address, U256)>, fatal_missing_code: FatalMissingCode, // called with the untrimmed tries after each batch - observer: &mut impl Observer, -) -> anyhow::Result>> + observer: &mut impl Observer, +) -> anyhow::Result>> where - StateTrieT::Key: Ord + From
, + WorldT::SubtriePath: Ord + From
, { - // Initialise the storage tries. - for (haddr, acct) in state_trie.iter() { - let storage = storage_tries.entry(haddr).or_insert({ - let mut it = StorageTrie::default(); - it.insert_hash(MptKey::default(), acct.storage_root) - .expect("empty trie insert cannot fail"); - it - }); - ensure!( - storage.root() == acct.storage_root, - "inconsistent initial storage for hashed address {haddr:x}" - ) - } - // These are the per-block tries. let mut transaction_trie = TransactionTrie::new(); let mut receipt_trie = ReceiptTrie::new(); @@ -441,26 +416,24 @@ where let mut batch_contract_code = BTreeSet::from([vec![]]); // always include empty code let mut before = IntraBlockTries { - state: state_trie.clone(), + world: world.clone(), transaction: transaction_trie.clone(), receipt: receipt_trie.clone(), - storage: storage_tries.clone(), }; // We want to perform mask the TrieInputs above, // but won't know the bounds until after the loop below, // so store that information here. let mut storage_masks = BTreeMap::<_, BTreeSet>::new(); - let mut state_mask = BTreeSet::::new(); + let mut state_mask = BTreeSet::::new(); if txn_ix == 0 { do_pre_execution( block, ger_data, - &mut storage_tries, &mut storage_masks, &mut state_mask, - &mut state_trie, + &mut world, )?; } @@ -510,28 +483,12 @@ where .map_err(|e| anyhow!("{e:?}")) .context(format!("couldn't decode receipt in txn {tx_hash:x}"))?; - let (mut acct, born) = state_trie - .get_by_address(addr) - .map(|acct| (acct, false)) - .unwrap_or((AccountRlp::default(), true)); + let born = !world.contains(addr)?; if born { // Empty accounts cannot have non-empty storage, // so we can safely insert a default trie. - storage_tries.insert(keccak_hash::keccak(addr), StorageTrie::default()); - } - - if born || just_access { - state_trie - .clone() - .insert_by_address(addr, acct) - .context(format!( - "couldn't reach state of {} address {addr:x} in txn {tx_hash:x}", - match born { - true => "created", - false => "accessed", - } - ))?; + world.create_storage(addr)? } let do_writes = !just_access @@ -551,71 +508,60 @@ where ); if do_writes { - acct.balance = balance.unwrap_or(acct.balance); - acct.nonce = nonce.unwrap_or(acct.nonce); - acct.code_hash = code_usage - .map(|it| match it { + if let Some(new) = balance { + world.update_balance(addr, |it| *it = new)? + } + if let Some(new) = nonce { + world.update_nonce(addr, |it| *it = new)? + } + if let Some(usage) = code_usage { + match usage { ContractCodeUsage::Read(hash) => { // TODO(Nashtare): https://github.com/0xPolygonZero/zk_evm/issues/700 - // This is a bug in the zero tracer, which shouldn't be giving us - // this read at all. Workaround for now. + // This is a bug in the zero tracer, + // which shouldn't be giving us this read at all. + // Workaround for now. + // The fix should involve removing the `Either` + // below. match (fatal_missing_code, code.get(hash)) { (FatalMissingCode(true), None) => { bail!("no code for hash {hash:x}") } (_, Some(byte_code)) => { + world.set_code(addr, Either::Left(&byte_code))?; batch_contract_code.insert(byte_code); } - (_, None) => { - log::warn!("no code for {hash:x}") - } + (_, None) => world.set_code(addr, Either::Right(hash))?, } - - anyhow::Ok(hash) } ContractCodeUsage::Write(bytes) => { code.insert(bytes.clone()); - let hash = keccak_hash::keccak(&bytes); + world.set_code(addr, Either::Left(&bytes))?; batch_contract_code.insert(bytes); - Ok(hash) } - }) - .transpose()? - .unwrap_or(acct.code_hash); - - if !storage_written.is_empty() { - let storage = match born { - true => storage_tries.entry(keccak_hash::keccak(addr)).or_default(), - false => storage_tries.get_mut(&keccak_hash::keccak(addr)).context( - format!( - "missing storage trie for address {addr:x} in txn {tx_hash:x}" - ), - )?, }; + } + if !storage_written.is_empty() { for (k, v) in storage_written { - let slot = MptKey::from_hash(keccak_hash::keccak(k)); match v.is_zero() { // this is actually a delete - true => storage_mask.extend(storage.reporting_remove(slot)?), - false => { - storage.insert(slot, rlp::encode(&v).to_vec())?; - } + true => storage_mask + .extend(world.reporting_destroy_slot(addr, k.into_uint())?), + false => world.store_int(addr, k.into_uint(), v)?, } } - acct.storage_root = storage.root(); } - state_trie.insert_by_address(addr, acct)?; - state_mask.insert(::from(addr)); + state_mask.insert(::from(addr)); } else { // Simple state access - state_mask.insert(::from(addr)); + state_mask.insert(::from(addr)); } if self_destructed { - storage_tries.remove(&keccak_hash::keccak(addr)); - state_mask.extend(state_trie.reporting_remove(addr)?) + world.destroy_storage(addr)?; + state_mask.extend(world.reporting_destroy(addr)?) } } @@ -633,41 +579,22 @@ where withdrawals: match loop_ix == loop_len { true => { for (addr, amt) in &withdrawals { - state_mask.insert(::from(*addr)); - let mut acct = state_trie - .get_by_address(*addr) - .context(format!("missing address {addr:x} for withdrawal"))?; - acct.balance += *amt; - state_trie - .insert_by_address(*addr, acct) - // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 - // Add an entry API - .expect("insert must succeed with the same key as a successful `get`"); + state_mask.insert(::from(*addr)); + world.update_balance(*addr, |it| *it += *amt)?; } mem::take(&mut withdrawals) } false => vec![], }, before: { - before.state.mask(state_mask)?; + before.world.mask(state_mask)?; before.receipt.mask(batch_first_txn_ix..txn_ix)?; before.transaction.mask(batch_first_txn_ix..txn_ix)?; - - let keep = storage_masks - .keys() - .map(keccak_hash::keccak) - .collect::>(); - before.storage.retain(|haddr, _| keep.contains(haddr)); - - for (addr, mask) in storage_masks { - if let Some(it) = before.storage.get_mut(&keccak_hash::keccak(addr)) { - it.mask(mask)? - } // else must have self-destructed - } + before.world.mask_storage(storage_masks)?; before }, after: TrieRoots { - state_root: state_trie.root(), + state_root: world.root(), transactions_root: transaction_trie.root(), receipts_root: receipt_trie.root(), }, @@ -676,8 +603,7 @@ where observer.collect_tries( block.block_number, batch_index, - &state_trie, - &storage_tries, + &world, &transaction_trie, &receipt_trie, ) @@ -687,38 +613,29 @@ where } /// Performs all the pre-txn execution rules of the targeted network. -fn do_pre_execution( +fn do_pre_execution( block: &BlockMetadata, ger_data: Option<(H256, H256)>, - storage: &mut BTreeMap, trim_storage: &mut BTreeMap>, - trim_state: &mut BTreeSet, - state_trie: &mut StateTrieT, + trim_state: &mut BTreeSet, + world: &mut WorldT, ) -> anyhow::Result<()> where - StateTrieT::Key: From
+ Ord, + WorldT::SubtriePath: From
+ Ord, { // Ethereum mainnet: EIP-4788 if cfg!(feature = "eth_mainnet") { return do_beacon_hook( block.block_timestamp, - storage, trim_storage, block.parent_beacon_block_root, trim_state, - state_trie, + world, ); } if cfg!(feature = "cdk_erigon") { - return do_scalable_hook( - block, - ger_data, - storage, - trim_storage, - trim_state, - state_trie, - ); + return do_scalable_hook(block, ger_data, trim_storage, trim_state, world); } Ok(()) @@ -729,37 +646,30 @@ where /// /// This is Polygon-CDK-specific, and runs at the start of the block, /// before any transactions (as per the Etrog specification). -fn do_scalable_hook( +fn do_scalable_hook( block: &BlockMetadata, ger_data: Option<(H256, H256)>, - storage: &mut BTreeMap, trim_storage: &mut BTreeMap>, - trim_state: &mut BTreeSet, - state_trie: &mut StateTrieT, + trim_state: &mut BTreeSet, + world: &mut WorldT, ) -> anyhow::Result<()> where - StateTrieT::Key: From
+ Ord, + WorldT::SubtriePath: From
+ Ord, { use evm_arithmetization::testing_utils::{ - ADDRESS_SCALABLE_L2, ADDRESS_SCALABLE_L2_ADDRESS_HASHED, GLOBAL_EXIT_ROOT_ADDRESS, - GLOBAL_EXIT_ROOT_ADDRESS_HASHED, GLOBAL_EXIT_ROOT_STORAGE_POS, LAST_BLOCK_STORAGE_POS, - STATE_ROOT_STORAGE_POS, TIMESTAMP_STORAGE_POS, + ADDRESS_SCALABLE_L2, GLOBAL_EXIT_ROOT_ADDRESS, GLOBAL_EXIT_ROOT_STORAGE_POS, + LAST_BLOCK_STORAGE_POS, STATE_ROOT_STORAGE_POS, TIMESTAMP_STORAGE_POS, }; if block.block_number.is_zero() { return Err(anyhow!("Attempted to prove the Genesis block!")); } - let scalable_storage = storage - .get_mut(&ADDRESS_SCALABLE_L2_ADDRESS_HASHED) - .context("missing scalable contract storage trie")?; let scalable_trim = trim_storage.entry(ADDRESS_SCALABLE_L2).or_default(); - let timestamp_slot_key = MptKey::from_slot_position(U256::from(TIMESTAMP_STORAGE_POS.1)); + let timestamp = world + .load_int(ADDRESS_SCALABLE_L2, U256::from(TIMESTAMP_STORAGE_POS.1)) + .unwrap_or_default(); - let timestamp = scalable_storage - .get(×tamp_slot_key) - .map(rlp::decode::) - .unwrap_or(Ok(0.into()))?; let timestamp = core::cmp::max(timestamp, block.block_timestamp); // Store block number and largest timestamp @@ -770,38 +680,31 @@ where ] { let slot = MptKey::from_slot_position(ix); - // These values are never 0. - scalable_storage.insert(slot, alloy::rlp::encode(u.compat()))?; + ensure!(!u.is_zero()); + world.store_int(ADDRESS_SCALABLE_L2, ix, u)?; scalable_trim.insert(slot); } // Store previous block root hash - let prev_block_root_hash = state_trie.root(); + let prev_block_root_hash = world.root(); let mut arr = [0; 64]; (block.block_number - 1).to_big_endian(&mut arr[0..32]); U256::from(STATE_ROOT_STORAGE_POS.1).to_big_endian(&mut arr[32..64]); let slot = MptKey::from_hash(keccak_hash::keccak(arr)); - scalable_storage.insert(slot, alloy::rlp::encode(prev_block_root_hash.compat()))?; + world.store_hash( + ADDRESS_SCALABLE_L2, + keccak_hash::keccak(arr), + prev_block_root_hash, + )?; + scalable_trim.insert(slot); - trim_state.insert(::from(ADDRESS_SCALABLE_L2)); - let mut scalable_acct = state_trie - .get_by_address(ADDRESS_SCALABLE_L2) - .context("missing scalable contract address")?; - scalable_acct.storage_root = scalable_storage.root(); - state_trie - .insert_by_address(ADDRESS_SCALABLE_L2, scalable_acct) - // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 - // Add an entry API - .expect("insert must succeed with the same key as a successful `get`"); + trim_state.insert(::from(ADDRESS_SCALABLE_L2)); // Update GER contract's storage if necessary if let Some((root, l1blockhash)) = ger_data { - let ger_storage = storage - .get_mut(&GLOBAL_EXIT_ROOT_ADDRESS_HASHED) - .context("missing GER contract storage trie")?; let ger_trim = trim_storage.entry(GLOBAL_EXIT_ROOT_ADDRESS).or_default(); let mut arr = [0; 64]; @@ -809,19 +712,14 @@ where U256::from(GLOBAL_EXIT_ROOT_STORAGE_POS.1).to_big_endian(&mut arr[32..64]); let slot = MptKey::from_hash(keccak_hash::keccak(arr)); - ger_storage.insert(slot, alloy::rlp::encode(l1blockhash.compat()))?; + world.store_hash( + GLOBAL_EXIT_ROOT_ADDRESS, + keccak_hash::keccak(arr), + l1blockhash, + )?; ger_trim.insert(slot); - trim_state.insert(::from(GLOBAL_EXIT_ROOT_ADDRESS)); - let mut ger_acct = state_trie - .get_by_address(GLOBAL_EXIT_ROOT_ADDRESS) - .context("missing GER contract address")?; - ger_acct.storage_root = ger_storage.root(); - state_trie - .insert_by_address(GLOBAL_EXIT_ROOT_ADDRESS, ger_acct) - // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 - // Add an entry API - .expect("insert must succeed with the same key as a successful `get`"); + trim_state.insert(::from(GLOBAL_EXIT_ROOT_ADDRESS)); } Ok(()) @@ -832,26 +730,22 @@ where /// /// This is Cancun-specific, and runs at the start of the block, /// before any transactions (as per the EIP). -fn do_beacon_hook( +fn do_beacon_hook( block_timestamp: U256, - storage: &mut BTreeMap, trim_storage: &mut BTreeMap>, parent_beacon_block_root: H256, - trim_state: &mut BTreeSet, - state_trie: &mut StateTrieT, + trim_state: &mut BTreeSet, + world: &mut WorldT, ) -> anyhow::Result<()> where - StateTrieT::Key: From
+ Ord, + WorldT::SubtriePath: From
+ Ord, { use evm_arithmetization::testing_utils::{ - BEACON_ROOTS_CONTRACT_ADDRESS, BEACON_ROOTS_CONTRACT_ADDRESS_HASHED, HISTORY_BUFFER_LENGTH, + BEACON_ROOTS_CONTRACT_ADDRESS, HISTORY_BUFFER_LENGTH, }; let timestamp_idx = block_timestamp % HISTORY_BUFFER_LENGTH.value; let root_idx = timestamp_idx + HISTORY_BUFFER_LENGTH.value; - let beacon_storage = storage - .get_mut(&BEACON_ROOTS_CONTRACT_ADDRESS_HASHED) - .context("missing beacon contract storage trie")?; let beacon_trim = trim_storage .entry(BEACON_ROOTS_CONTRACT_ADDRESS) .or_default(); @@ -867,23 +761,16 @@ where beacon_trim.insert(slot); match u.is_zero() { - true => beacon_trim.extend(beacon_storage.reporting_remove(slot)?), + true => { + beacon_trim.extend(world.reporting_destroy_slot(BEACON_ROOTS_CONTRACT_ADDRESS, ix)?) + } false => { - beacon_storage.insert(slot, alloy::rlp::encode(u.compat()))?; + world.store_int(BEACON_ROOTS_CONTRACT_ADDRESS, ix, u)?; beacon_trim.insert(slot); } } } - trim_state.insert(::from(BEACON_ROOTS_CONTRACT_ADDRESS)); - let mut beacon_acct = state_trie - .get_by_address(BEACON_ROOTS_CONTRACT_ADDRESS) - .context("missing beacon contract address")?; - beacon_acct.storage_root = beacon_storage.root(); - state_trie - .insert_by_address(BEACON_ROOTS_CONTRACT_ADDRESS, beacon_acct) - // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 - // Add an entry API - .expect("insert must succeed with the same key as a successful `get`"); + trim_state.insert(::from(BEACON_ROOTS_CONTRACT_ADDRESS)); Ok(()) } diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index 057d11e89..1f1c87888 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -1,8 +1,10 @@ //! An _Ethereum Node_ executes _transactions_ in _blocks_. //! //! Execution mutates two key data structures: -//! - [The state trie](https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/#state-trie). -//! - [The storage tries](https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/#storage-trie). +//! - [The state](https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/#state-trie), +//! which tracks, e.g the account balance. +//! - [The storage](https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/#storage-trie), +//! which is a huge array of integers, per-account. //! //! Ethereum nodes expose information about the transactions over RPC, e.g: //! - [The specific changes to the storage tries](TxnTrace::storage_written). @@ -13,7 +15,8 @@ //! //! **Prover perfomance is a high priority.** //! -//! The aformentioned trie structures may have subtries _hashed out_. +//! The aformentioned data structures are represented as tries, +//! which may have subtries _hashed out_. //! That is, any node (and its children!) may be replaced by its hash, //! while maintaining provability of its contents: //! @@ -44,12 +47,16 @@ /// Over RPC, ethereum nodes expose their tries as a series of binary /// [`wire::Instruction`]s in a node-dependant format. /// -/// These are parsed into the relevant trie depending on the node: +/// These are parsed into the relevant state and storage data structures, +/// depending on the node: /// - [`type2`], which contains an [`smt_trie`]. /// - [`type1`], which contains an [`mpt_trie`]. /// /// After getting the tries, /// we can continue to do the main work of "executing" the transactions. +/// +/// The core of this library is agnostic over the (combined) +/// state and storage representation - see [`world::World`] for more. const _DEVELOPER_DOCS: () = (); mod interface; @@ -60,6 +67,7 @@ mod tries; mod type1; mod type2; mod wire; +mod world; pub use core::{entrypoint, WireDisposition}; diff --git a/trace_decoder/src/observer.rs b/trace_decoder/src/observer.rs index f9811e87c..428cac086 100644 --- a/trace_decoder/src/observer.rs +++ b/trace_decoder/src/observer.rs @@ -1,15 +1,14 @@ -use std::collections::BTreeMap; use std::marker::PhantomData; -use ethereum_types::{H256, U256}; +use ethereum_types::U256; use crate::core::IntraBlockTries; -use crate::tries::{ReceiptTrie, StorageTrie, TransactionTrie}; +use crate::tries::{ReceiptTrie, TransactionTrie}; /// Observer API for the trace decoder. /// Observer is used to collect various debugging and metadata info /// from the trace decoder run. -pub trait Observer { +pub trait Observer { /// Collect tries after the transaction/batch execution. /// /// Passing the arguments one by one through reference, because @@ -19,8 +18,7 @@ pub trait Observer { &mut self, block: U256, batch: usize, - state_trie: &StateTrieT, - storage: &BTreeMap, + state_trie: &WorldT, transaction_trie: &TransactionTrie, receipt_trie: &ReceiptTrie, ); @@ -55,13 +53,12 @@ impl TriesObserver { } } -impl Observer for TriesObserver { +impl Observer for TriesObserver { fn collect_tries( &mut self, block: U256, batch: usize, - state_trie: &StateTrieT, - storage: &BTreeMap, + state_trie: &WorldT, transaction_trie: &TransactionTrie, receipt_trie: &ReceiptTrie, ) { @@ -69,8 +66,7 @@ impl Observer for TriesObserver { block, batch, tries: IntraBlockTries { - state: state_trie.clone(), - storage: storage.clone(), + world: state_trie.clone(), transaction: transaction_trie.clone(), receipt: receipt_trie.clone(), }, @@ -99,13 +95,12 @@ impl DummyObserver { } } -impl Observer for DummyObserver { +impl Observer for DummyObserver { fn collect_tries( &mut self, _block: U256, _batch: usize, - _state_trie: &StateTrieT, - _storage: &BTreeMap, + _state_trie: &WorldT, _transaction_trie: &TransactionTrie, _receipt_trie: &ReceiptTrie, ) { diff --git a/trace_decoder/src/tries.rs b/trace_decoder/src/tries.rs index 91add4d98..7da8d2cfa 100644 --- a/trace_decoder/src/tries.rs +++ b/trace_decoder/src/tries.rs @@ -1,103 +1,18 @@ -//! Principled trie types and abstractions used in this library. +//! Principled trie types used in this library. use core::fmt; -use std::{cmp, collections::BTreeMap, marker::PhantomData}; +use std::cmp; use anyhow::ensure; use bitvec::{array::BitArray, slice::BitSlice}; use copyvec::CopyVec; -use ethereum_types::{Address, BigEndianHash as _, H256, U256}; +use ethereum_types::{Address, H256, U256}; use evm_arithmetization::generation::mpt::AccountRlp; use mpt_trie::partial_trie::{HashedPartialTrie, Node, OnOrphanedHashNode, PartialTrie as _}; use u4::{AsNibbles, U4}; -/// See . -/// -/// Portions of the trie may be _hashed out_: see [`Self::insert_hash`]. -#[derive(Debug, Clone, PartialEq, Eq)] -struct TypedMpt { - inner: HashedPartialTrie, - _ty: PhantomData T>, -} - -impl TypedMpt { - const PANIC_MSG: &str = "T encoding/decoding should round-trip,\ - and only encoded `T`s are ever inserted"; - fn new() -> Self { - Self { - inner: HashedPartialTrie::new(Node::Empty), - _ty: PhantomData, - } - } - /// Insert a node which represents an out-of-band sub-trie. - /// - /// See [module documentation](super) for more. - fn insert_hash(&mut self, key: MptKey, hash: H256) -> anyhow::Result<()> { - self.inner.insert(key.into_nibbles(), hash)?; - Ok(()) - } - /// Returns [`Err`] if the `key` crosses into a part of the trie that - /// is hashed out. - fn insert(&mut self, key: MptKey, value: T) -> anyhow::Result<()> - where - T: rlp::Encodable + rlp::Decodable, - { - self.inner - .insert(key.into_nibbles(), rlp::encode(&value).to_vec())?; - Ok(()) - } - /// Note that this returns [`None`] if `key` crosses into a part of the - /// trie that is hashed out. - /// - /// # Panics - /// - If [`rlp::decode`]-ing for `T` doesn't round-trip. - fn get(&self, key: MptKey) -> Option - where - T: rlp::Decodable, - { - let bytes = self.inner.get(key.into_nibbles())?; - Some(rlp::decode(bytes).expect(Self::PANIC_MSG)) - } - const fn as_hashed_partial_trie(&self) -> &HashedPartialTrie { - &self.inner - } - fn as_mut_hashed_partial_trie_unchecked(&mut self) -> &mut HashedPartialTrie { - &mut self.inner - } - fn root(&self) -> H256 { - self.inner.hash() - } - /// Note that this returns owned paths and items. - fn iter(&self) -> impl Iterator + '_ - where - T: rlp::Decodable, - { - self.inner.keys().filter_map(|nib| { - let path = MptKey::from_nibbles(nib); - Some((path, self.get(path)?)) - }) - } -} - -impl Default for TypedMpt { - fn default() -> Self { - Self::new() - } -} - -impl<'a, T> IntoIterator for &'a TypedMpt -where - T: rlp::Decodable, -{ - type Item = (MptKey, T); - type IntoIter = Box + 'a>; - fn into_iter(self) -> Self::IntoIter { - Box::new(self.iter()) - } -} - /// Bounded sequence of [`U4`], -/// used as a key for [`TypedMpt`]. +/// used as a key for [MPT](HashedPartialTrie) types in this module. /// /// Semantically equivalent to [`mpt_trie::nibbles::Nibbles`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] @@ -116,17 +31,6 @@ impl MptKey { pub fn new(components: impl IntoIterator) -> anyhow::Result { Ok(MptKey(CopyVec::try_from_iter(components)?)) } - pub fn into_hash_left_padded(mut self) -> H256 { - for _ in 0..self.0.spare_capacity_mut().len() { - self.0.insert(0, U4::Dec00) - } - let mut packed = [0u8; 32]; - AsNibbles(&mut packed).pack_from_slice(&self.0); - H256::from_slice(&packed) - } - pub fn from_address(address: Address) -> Self { - Self::from_hash(keccak_hash::keccak(address)) - } pub fn from_slot_position(pos: U256) -> Self { let mut bytes = [0; 32]; pos.to_big_endian(&mut bytes); @@ -189,7 +93,7 @@ fn mpt_key_into_hash() { } /// Bounded sequence of bits, -/// used as a key for [`StateSmt`]. +/// used as a key for SMT tries. /// /// Semantically equivalent to [`smt_trie::bits::Bits`]. #[derive(Clone, Copy)] @@ -366,246 +270,90 @@ impl From for HashedPartialTrie { } } -/// TODO(0xaatif): document this after refactoring is done -pub trait StateTrie { - type Key; - fn insert_by_address(&mut self, address: Address, account: AccountRlp) -> anyhow::Result<()>; - fn get_by_address(&self, address: Address) -> Option; - fn reporting_remove(&mut self, address: Address) -> anyhow::Result>; - /// _Hash out_ parts of the trie that aren't in `addresses`. - fn mask(&mut self, address: impl IntoIterator) -> anyhow::Result<()>; - fn iter(&self) -> impl Iterator + '_; - fn root(&self) -> H256; -} - /// Global, [`Address`] `->` [`AccountRlp`]. /// /// See -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct StateMpt { - typed: TypedMpt, + /// Values are always [`rlp`]-encoded [`AccountRlp`], + /// inserted at [256 bits](MptKey::from_hash). + inner: HashedPartialTrie, +} + +impl Default for StateMpt { + fn default() -> Self { + Self::new() + } +} + +#[track_caller] +fn assert_rlp_account(bytes: impl AsRef<[u8]>) -> AccountRlp { + rlp::decode(bytes.as_ref()).expect("invalid RLP in StateMPT") } impl StateMpt { - pub fn new(strategy: OnOrphanedHashNode) -> Self { + pub fn new() -> Self { Self { - typed: TypedMpt { - inner: HashedPartialTrie::new_with_strategy(Node::Empty, strategy), - _ty: PhantomData, - }, + inner: HashedPartialTrie::new_with_strategy( + Node::Empty, + // This frontend is intended to be used with our custom `zeroTracer`, + // which covers branch-to-extension collapse edge cases. + OnOrphanedHashNode::CollapseToExtension, + ), } } - /// Insert a _hashed out_ part of the trie - pub fn insert_hash_by_key(&mut self, key: MptKey, hash: H256) -> anyhow::Result<()> { - self.typed.insert_hash(key, hash) - } - #[deprecated = "prefer operations on `Address` where possible, as SMT support requires this"] - pub fn insert_by_hashed_address( - &mut self, - key: H256, - account: AccountRlp, - ) -> anyhow::Result<()> { - self.typed.insert(MptKey::from_hash(key), account) + pub fn as_hashed_partial_trie(&self) -> &HashedPartialTrie { + &self.inner } - pub fn iter(&self) -> impl Iterator + '_ { - self.typed - .iter() - .map(|(key, rlp)| (key.into_hash().expect("key is always H256"), rlp)) + /// Insert a _hashed out_ part of the trie + pub fn insert_hash(&mut self, key: MptKey, hash: H256) -> anyhow::Result<()> { + Ok(self.inner.insert(key.into_nibbles(), hash)?) } - pub fn as_hashed_partial_trie(&self) -> &mpt_trie::partial_trie::HashedPartialTrie { - self.typed.as_hashed_partial_trie() + pub fn insert(&mut self, key: H256, account: AccountRlp) -> anyhow::Result<()> { + Ok(self.inner.insert( + MptKey::from_hash(key).into_nibbles(), + rlp::encode(&account).to_vec(), + )?) } -} - -impl StateTrie for StateMpt { - type Key = MptKey; - fn insert_by_address(&mut self, address: Address, account: AccountRlp) -> anyhow::Result<()> { - #[expect(deprecated)] - self.insert_by_hashed_address(keccak_hash::keccak(address), account) + pub fn get(&self, key: H256) -> Option { + self.inner + .get(MptKey::from_hash(key).into_nibbles()) + .map(assert_rlp_account) } - fn get_by_address(&self, address: Address) -> Option { - self.typed - .get(MptKey::from_hash(keccak_hash::keccak(address))) + pub fn root(&self) -> H256 { + self.inner.hash() } - /// Delete the account at `address`, returning any remaining branch on - /// collapse - fn reporting_remove(&mut self, address: Address) -> anyhow::Result> { + pub fn reporting_remove(&mut self, address: Address) -> anyhow::Result> { delete_node_and_report_remaining_key_if_branch_collapsed( - self.typed.as_mut_hashed_partial_trie_unchecked(), - MptKey::from_address(address), + &mut self.inner, + MptKey::from_hash(keccak_hash::keccak(address)), ) } - fn mask(&mut self, addresses: impl IntoIterator) -> anyhow::Result<()> { - let inner = mpt_trie::trie_subsets::create_trie_subset( - self.typed.as_hashed_partial_trie(), + pub fn mask(&mut self, addresses: impl IntoIterator) -> anyhow::Result<()> { + let new = mpt_trie::trie_subsets::create_trie_subset( + &self.inner, addresses.into_iter().map(MptKey::into_nibbles), )?; - self.typed = TypedMpt { - inner, - _ty: PhantomData, - }; + self.inner = new; Ok(()) } - fn iter(&self) -> impl Iterator + '_ { - self.typed - .iter() - .map(|(key, rlp)| (key.into_hash().expect("key is always H256"), rlp)) - } - fn root(&self) -> H256 { - self.typed.root() + pub fn iter(&self) -> impl Iterator + '_ { + self.inner.items().filter_map(|(key, rlp)| match rlp { + mpt_trie::trie_ops::ValOrHash::Val(vec) => Some(( + MptKey::from_nibbles(key).into_hash().expect("bad depth"), + assert_rlp_account(vec), + )), + mpt_trie::trie_ops::ValOrHash::Hash(_) => None, + }) } } impl From for HashedPartialTrie { - fn from(value: StateMpt) -> Self { - let StateMpt { - typed: TypedMpt { inner, _ty }, - } = value; + fn from(StateMpt { inner }: StateMpt) -> Self { inner } } -// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/706 -// We're covering for [`smt_trie`] in a couple of ways: -// - insertion operations aren't fallible, they just panic. -// - it documents a requirement that `set_hash` is called before `set`. -#[derive(Clone, Debug)] -pub struct StateSmt { - address2state: BTreeMap, - hashed_out: BTreeMap, -} - -impl StateTrie for StateSmt { - type Key = SmtKey; - fn insert_by_address(&mut self, address: Address, account: AccountRlp) -> anyhow::Result<()> { - self.address2state.insert(address, account); - Ok(()) - } - fn get_by_address(&self, address: Address) -> Option { - self.address2state.get(&address).copied() - } - fn reporting_remove(&mut self, address: Address) -> anyhow::Result> { - self.address2state.remove(&address); - Ok(None) - } - fn mask(&mut self, address: impl IntoIterator) -> anyhow::Result<()> { - let _ = address; - Ok(()) - } - fn iter(&self) -> impl Iterator + '_ { - self.address2state - .iter() - .map(|(addr, acct)| (keccak_hash::keccak(addr), *acct)) - } - fn root(&self) -> H256 { - conv_hash::smt2eth(self.as_smt().root) - } -} - -impl StateSmt { - pub(crate) fn new_unchecked( - address2state: BTreeMap, - hashed_out: BTreeMap, - ) -> Self { - Self { - address2state, - hashed_out, - } - } - - fn as_smt(&self) -> smt_trie::smt::Smt { - let Self { - address2state, - hashed_out, - } = self; - let mut smt = smt_trie::smt::Smt::::default(); - for (k, v) in hashed_out { - smt.set_hash(k.into_smt_bits(), conv_hash::eth2smt(*v)); - } - for ( - addr, - AccountRlp { - nonce, - balance, - storage_root, - code_hash, - }, - ) in address2state - { - smt.set(smt_trie::keys::key_nonce(*addr), *nonce); - smt.set(smt_trie::keys::key_balance(*addr), *balance); - smt.set(smt_trie::keys::key_code(*addr), code_hash.into_uint()); - smt.set( - // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/707 - // combined abstraction for state and storage - smt_trie::keys::key_storage(*addr, U256::zero()), - storage_root.into_uint(), - ); - } - smt - } -} - -mod conv_hash { - //! We [`u64::to_le_bytes`] because: - //! - Reference go code just puns the bytes: - //! - It's better to fix the endianness for correctness. - //! - Most (consumer) CPUs are little-endian. - - use std::array; - - use ethereum_types::H256; - use itertools::Itertools as _; - use plonky2::{ - field::{ - goldilocks_field::GoldilocksField, - types::{Field as _, PrimeField64}, - }, - hash::hash_types::HashOut, - }; - - /// # Panics - /// - On certain inputs if `debug_assertions` are enabled. See - /// [`GoldilocksField::from_canonical_u64`] for more. - pub fn eth2smt(H256(bytes): H256) -> smt_trie::smt::HashOut { - let mut bytes = bytes.into_iter(); - // (no unsafe, no unstable) - let ret = HashOut { - elements: array::from_fn(|_ix| { - let (a, b, c, d, e, f, g, h) = bytes.next_tuple().unwrap(); - GoldilocksField::from_canonical_u64(u64::from_le_bytes([a, b, c, d, e, f, g, h])) - }), - }; - assert_eq!(bytes.len(), 0); - ret - } - pub fn smt2eth(HashOut { elements }: smt_trie::smt::HashOut) -> H256 { - H256( - build_array::ArrayBuilder::from_iter( - elements - .iter() - .map(GoldilocksField::to_canonical_u64) - .flat_map(u64::to_le_bytes), - ) - .build_exact() - .unwrap(), - ) - } - - #[test] - fn test() { - use plonky2::field::types::Field64 as _; - let mut max = std::iter::repeat(GoldilocksField::ORDER - 1).flat_map(u64::to_le_bytes); - for h in [ - H256::zero(), - H256(array::from_fn(|ix| ix as u8)), - H256(array::from_fn(|_| max.next().unwrap())), - ] { - assert_eq!(smt2eth(eth2smt(h)), h); - } - } -} - /// Global, per-account. /// /// See @@ -622,10 +370,9 @@ impl StorageTrie { pub fn get(&mut self, key: &MptKey) -> Option<&[u8]> { self.untyped.get(key.into_nibbles()) } - pub fn insert(&mut self, key: MptKey, value: Vec) -> anyhow::Result>> { - let prev = self.get(&key).map(Vec::from); + pub fn insert(&mut self, key: MptKey, value: Vec) -> anyhow::Result<()> { self.untyped.insert(key.into_nibbles(), value)?; - Ok(prev) + Ok(()) } pub fn insert_hash(&mut self, key: MptKey, hash: H256) -> anyhow::Result<()> { self.untyped.insert(key.into_nibbles(), hash)?; diff --git a/trace_decoder/src/type1.rs b/trace_decoder/src/type1.rs index c44beaec7..c982a1ab3 100644 --- a/trace_decoder/src/type1.rs +++ b/trace_decoder/src/type1.rs @@ -15,25 +15,13 @@ use u4::U4; use crate::tries::{MptKey, StateMpt, StorageTrie}; use crate::wire::{Instruction, SmtLeaf}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct Frontend { pub state: StateMpt, pub code: BTreeSet>>, pub storage: BTreeMap, } -impl Default for Frontend { - // This frontend is intended to be used with our custom `zeroTracer`, - // which covers branch-to-extension collapse edge cases. - fn default() -> Self { - Self { - state: StateMpt::new(OnOrphanedHashNode::CollapseToExtension), - code: BTreeSet::new(), - storage: BTreeMap::new(), - } - } -} - pub fn frontend(instructions: impl IntoIterator) -> anyhow::Result { let executions = execute(instructions)?; ensure!( @@ -66,7 +54,7 @@ fn visit( Node::Hash(Hash { raw_hash }) => { frontend .state - .insert_hash_by_key(MptKey::new(path.iter().copied())?, raw_hash.into())?; + .insert_hash(MptKey::new(path.iter().copied())?, raw_hash.into())?; } Node::Leaf(Leaf { key, value }) => { let path = MptKey::new(path.iter().copied().chain(key))? @@ -105,8 +93,7 @@ fn visit( } }, }; - #[expect(deprecated)] // this is MPT-specific code - frontend.state.insert_by_hashed_address(path, account)?; + frontend.state.insert(path, account)?; } } } @@ -379,8 +366,6 @@ fn finish_stack(v: &mut Vec) -> anyhow::Result { #[test] fn test_tries() { - use crate::tries::StateTrie as _; - for (ix, case) in serde_json::from_str::>(include_str!("cases/zero_jerigon.json")) .unwrap() @@ -393,7 +378,7 @@ fn test_tries() { assert_eq!(case.expected_state_root, frontend.state.root()); for (haddr, acct) in frontend.state.iter() { - if acct.storage_root != StateMpt::default().root() { + if acct.storage_root != StorageTrie::default().root() { assert!(frontend.storage.contains_key(&haddr)) } } diff --git a/trace_decoder/src/type2.rs b/trace_decoder/src/type2.rs index 44d13e89a..845260d47 100644 --- a/trace_decoder/src/type2.rs +++ b/trace_decoder/src/type2.rs @@ -5,29 +5,19 @@ use std::collections::{BTreeMap, HashSet}; use anyhow::{bail, ensure, Context as _}; use ethereum_types::{Address, U256}; -use evm_arithmetization::generation::mpt::AccountRlp; use itertools::EitherOrBoth; use keccak_hash::H256; use nunny::NonEmpty; use stackstack::Stack; use crate::{ - tries::{SmtKey, StateSmt}, + tries::SmtKey, wire::{Instruction, SmtLeaf, SmtLeafType}, + world::{Type2Entry, Type2World}, }; -/// Combination of all the [`SmtLeaf::node_type`]s -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] -pub struct CollatedLeaf { - pub balance: Option, - pub nonce: Option, - pub code: Option, - pub code_length: Option, - pub storage: BTreeMap, -} - pub struct Frontend { - pub trie: StateSmt, + pub world: Type2World, pub code: HashSet>>, } @@ -36,13 +26,14 @@ pub struct Frontend { /// NOT call this function on untrusted inputs. pub fn frontend(instructions: impl IntoIterator) -> anyhow::Result { let (node, code) = fold(instructions).context("couldn't fold smt from instructions")?; - let trie = node2trie(node).context("couldn't construct trie and collation from folded node")?; - Ok(Frontend { trie, code }) + let world = + node2world(node).context("couldn't construct trie and collation from folded node")?; + Ok(Frontend { world, code }) } /// Node in a binary (SMT) tree. /// -/// This is an intermediary type on the way to [`StateSmt`]. +/// This is an intermediary type on the way to [`Type2World`]. enum Node { Branch(EitherOrBoth>), Hash([u8; 32]), @@ -113,45 +104,16 @@ fn fold1(instructions: impl IntoIterator) -> anyhow::Result< } } -fn node2trie(node: Node) -> anyhow::Result { +fn node2world(node: Node) -> anyhow::Result { let mut hashes = BTreeMap::new(); let mut leaves = BTreeMap::new(); visit(&mut hashes, &mut leaves, Stack::new(), node)?; - Ok(StateSmt::new_unchecked( - leaves - .into_iter() - .map( - |( - addr, - CollatedLeaf { - balance, - nonce, - // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/707 - // we shouldn't ignore these fields - code: _, - code_length: _, - storage: _, - }, - )| { - ( - addr, - AccountRlp { - nonce: nonce.unwrap_or_default(), - balance: balance.unwrap_or_default(), - storage_root: H256::zero(), - code_hash: H256::zero(), - }, - ) - }, - ) - .collect(), - hashes, - )) + Ok(Type2World::new_unchecked(leaves, hashes)) } fn visit( hashes: &mut BTreeMap, - leaves: &mut BTreeMap, + leaves: &mut BTreeMap, path: Stack, node: Node, ) -> anyhow::Result<()> { @@ -213,65 +175,7 @@ fn visit( #[test] fn test_tries() { - type Smt = smt_trie::smt::Smt; - use ethereum_types::BigEndianHash as _; - use plonky2::field::types::{Field, Field64 as _}; - - // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/707 - // this logic should live in StateSmt, but we need to - // - abstract over state and storage tries - // - parameterize the account types - // we preserve this code as a tested record of how it _should_ - // be done. - fn node2trie(node: Node) -> anyhow::Result { - let mut trie = Smt::default(); - let mut hashes = BTreeMap::new(); - let mut leaves = BTreeMap::new(); - visit(&mut hashes, &mut leaves, Stack::new(), node)?; - for (key, hash) in hashes { - trie.set_hash( - key.into_smt_bits(), - smt_trie::smt::HashOut { - elements: { - let ethereum_types::U256(arr) = hash.into_uint(); - for u in arr { - ensure!(u < smt_trie::smt::F::ORDER); - } - arr.map(smt_trie::smt::F::from_canonical_u64) - }, - }, - ); - } - for ( - addr, - CollatedLeaf { - balance, - nonce, - code, - code_length, - storage, - }, - ) in leaves - { - use smt_trie::keys::{key_balance, key_code, key_code_length, key_nonce, key_storage}; - - for (value, key_fn) in [ - (balance, key_balance as fn(_) -> _), - (nonce, key_nonce), - (code, key_code), - (code_length, key_code_length), - ] { - if let Some(value) = value { - trie.set(key_fn(addr), value); - } - } - for (slot, value) in storage { - trie.set(key_storage(addr, slot), value); - } - } - Ok(trie) - } - + use crate::world::World as _; for (ix, case) in serde_json::from_str::>(include_str!("cases/hermez_cdk_erigon.json")) .unwrap() @@ -279,13 +183,7 @@ fn test_tries() { .enumerate() { println!("case {}", ix); - let instructions = crate::wire::parse(&case.bytes).unwrap(); - let (node, _code) = fold(instructions).unwrap(); - let trie = node2trie(node).unwrap(); - assert_eq!(case.expected_state_root, { - let mut it = [0; 32]; - smt_trie::utils::hashout2u(trie.root).to_big_endian(&mut it); - ethereum_types::H256(it) - }); + let mut frontend = frontend(crate::wire::parse(&case.bytes).unwrap()).unwrap(); + assert_eq!(case.expected_state_root, frontend.world.root()); } } diff --git a/trace_decoder/src/world.rs b/trace_decoder/src/world.rs new file mode 100644 index 000000000..fa68854e4 --- /dev/null +++ b/trace_decoder/src/world.rs @@ -0,0 +1,420 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use alloy_compat::Compat as _; +use anyhow::{ensure, Context as _}; +use either::Either; +use ethereum_types::{Address, BigEndianHash as _, U256}; +use keccak_hash::H256; + +use crate::tries::{MptKey, SmtKey, StateMpt, StorageTrie}; + +/// The [core](crate::core) of this crate is agnostic over state and storage +/// representations. +/// +/// This is the common interface to those data structures. +/// See also [crate::_DEVELOPER_DOCS]. +pub(crate) trait World { + /// (State) subtries may be _hashed out. + /// This type is a key which may identify a subtrie. + type SubtriePath; + + ////////////////////// + /// Account operations + ////////////////////// + + /// Whether the state contains an account at the given address. + /// + /// `false` is not necessarily definitive - the address may belong to a + /// _hashed out_ subtrie. + fn contains(&mut self, address: Address) -> anyhow::Result; + + /// Update the balance for the account at the given address. + /// + /// Creates a new account at `address` if it does not exist. + fn update_balance(&mut self, address: Address, f: impl FnOnce(&mut U256)) + -> anyhow::Result<()>; + + /// Update the nonce for the account at the given address. + /// + /// Creates a new account at `address` if it does not exist. + fn update_nonce(&mut self, address: Address, f: impl FnOnce(&mut U256)) -> anyhow::Result<()>; + + /// Update the code for the account at the given address. + /// + /// Creates a new account at `address` if it does not exist. + fn set_code(&mut self, address: Address, code: Either<&[u8], H256>) -> anyhow::Result<()>; + + /// The [core](crate::core) of this crate tracks required subtries for + /// proving. + /// + /// In case of a state delete, it may be that certain parts of the subtrie + /// must be retained. If so, it will be returned as [`Some`]. + fn reporting_destroy(&mut self, address: Address) -> anyhow::Result>; + + ////////////////////// + /// Storage operations + ////////////////////// + + /// Create an account at the given address. + /// + /// It may not be an error if the address already exists. + fn create_storage(&mut self, address: Address) -> anyhow::Result<()>; + + /// Destroy storage for the given address' account. + fn destroy_storage(&mut self, address: Address) -> anyhow::Result<()>; + + /// Store an integer for the given account at the given `slot`. + fn store_int(&mut self, address: Address, slot: U256, value: U256) -> anyhow::Result<()>; + fn store_hash(&mut self, address: Address, hash: H256, value: H256) -> anyhow::Result<()>; + + /// Load an integer from the given account at the given `slot`. + fn load_int(&mut self, address: Address, slot: U256) -> anyhow::Result; + + /// Delete the given slot from the given account's storage. + /// + /// In case of a delete, it may be that certain parts of the subtrie + /// must be retained. If so, it will be returned as [`Some`]. + fn reporting_destroy_slot( + &mut self, + address: Address, + slot: U256, + ) -> anyhow::Result>; + fn mask_storage(&mut self, masks: BTreeMap>) -> anyhow::Result<()>; + + //////////////////// + /// Other operations + //////////////////// + + /// _Hash out_ parts of the (state) trie that aren't in `paths`. + fn mask(&mut self, paths: impl IntoIterator) -> anyhow::Result<()>; + + /// Return an identifier for the world. + fn root(&mut self) -> H256; +} + +#[derive(Clone, Debug)] +pub struct Type1World { + state: StateMpt, + /// Writes to storage should be reconciled with + /// [`storage_root`](evm_arithmetization::generation::mpt::AccountRlp)s. + storage: BTreeMap, +} + +impl Type1World { + pub fn new(state: StateMpt, mut storage: BTreeMap) -> anyhow::Result { + // Initialise the storage tries. + for (haddr, acct) in state.iter() { + let storage = storage.entry(haddr).or_insert_with(|| { + let mut it = StorageTrie::default(); + it.insert_hash(MptKey::default(), acct.storage_root) + .expect("empty trie insert cannot fail"); + it + }); + ensure!( + storage.root() == acct.storage_root, + "inconsistent initial storage for hashed address {haddr}" + ) + } + Ok(Self { state, storage }) + } + pub fn state_trie(&self) -> &mpt_trie::partial_trie::HashedPartialTrie { + self.state.as_hashed_partial_trie() + } + pub fn into_state_and_storage(self) -> (StateMpt, BTreeMap) { + let Self { state, storage } = self; + (state, storage) + } + fn get_storage_mut(&mut self, address: Address) -> anyhow::Result<&mut StorageTrie> { + self.storage + .get_mut(&keccak_hash::keccak(address)) + .context("no such storage") + } + fn on_storage( + &mut self, + address: Address, + f: impl FnOnce(&mut StorageTrie) -> anyhow::Result, + ) -> anyhow::Result { + let mut acct = self + .state + .get(keccak_hash::keccak(address)) + .context("no such account")?; + let storage = self.get_storage_mut(address)?; + let ret = f(storage)?; + acct.storage_root = storage.root(); + self.state.insert(keccak_hash::keccak(address), acct)?; + Ok(ret) + } +} + +impl World for Type1World { + type SubtriePath = MptKey; + fn contains(&mut self, address: Address) -> anyhow::Result { + Ok(self.state.get(keccak_hash::keccak(address)).is_some()) + } + fn update_balance( + &mut self, + address: Address, + f: impl FnOnce(&mut U256), + ) -> anyhow::Result<()> { + let key = keccak_hash::keccak(address); + let mut acct = self.state.get(key).unwrap_or_default(); + f(&mut acct.balance); + self.state.insert(key, acct) + } + fn update_nonce(&mut self, address: Address, f: impl FnOnce(&mut U256)) -> anyhow::Result<()> { + let key = keccak_hash::keccak(address); + let mut acct = self.state.get(key).unwrap_or_default(); + f(&mut acct.nonce); + self.state.insert(key, acct) + } + fn set_code(&mut self, address: Address, code: Either<&[u8], H256>) -> anyhow::Result<()> { + let key = keccak_hash::keccak(address); + let mut acct = self.state.get(key).unwrap_or_default(); + acct.code_hash = code.right_or_else(keccak_hash::keccak); + self.state.insert(key, acct) + } + fn reporting_destroy(&mut self, address: Address) -> anyhow::Result> { + self.state.reporting_remove(address) + } + fn mask( + &mut self, + addresses: impl IntoIterator, + ) -> anyhow::Result<()> { + self.state.mask(addresses) + } + fn root(&mut self) -> H256 { + self.state.root() + } + fn create_storage(&mut self, address: Address) -> anyhow::Result<()> { + let _clobbered = self + .storage + .insert(keccak_hash::keccak(address), StorageTrie::default()); + // ensure!(_clobbered.is_none()); // TODO(0xaatif): fails our tests + Ok(()) + } + fn destroy_storage(&mut self, address: Address) -> anyhow::Result<()> { + let removed = self.storage.remove(&keccak_hash::keccak(address)); + ensure!(removed.is_some()); + Ok(()) + } + + fn store_int(&mut self, address: Address, slot: U256, value: U256) -> anyhow::Result<()> { + self.on_storage(address, |it| { + it.insert( + MptKey::from_slot_position(slot), + alloy::rlp::encode(value.compat()), + ) + }) + } + + fn store_hash(&mut self, address: Address, hash: H256, value: H256) -> anyhow::Result<()> { + self.on_storage(address, |it| { + it.insert(MptKey::from_hash(hash), alloy::rlp::encode(value.compat())) + }) + } + + fn load_int(&mut self, address: Address, slot: U256) -> anyhow::Result { + let bytes = self + .get_storage_mut(address)? + .get(&MptKey::from_slot_position(slot)) + .context(format!("no storage at slot {slot} for address {address:x}"))?; + Ok(rlp::decode(bytes)?) + } + + fn reporting_destroy_slot( + &mut self, + address: Address, + slot: U256, + ) -> anyhow::Result> { + self.on_storage(address, |it| { + it.reporting_remove(MptKey::from_slot_position(slot)) + }) + } + + fn mask_storage(&mut self, masks: BTreeMap>) -> anyhow::Result<()> { + let keep = masks + .keys() + .map(keccak_hash::keccak) + .collect::>(); + self.storage.retain(|haddr, _| keep.contains(haddr)); + for (addr, mask) in masks { + if let Some(it) = self.storage.get_mut(&keccak_hash::keccak(addr)) { + it.mask(mask)? + } + } + Ok(()) + } +} + +impl World for Type2World { + type SubtriePath = SmtKey; + fn contains(&mut self, address: Address) -> anyhow::Result { + Ok(self.accounts.contains_key(&address)) + } + fn update_balance( + &mut self, + address: Address, + f: impl FnOnce(&mut U256), + ) -> anyhow::Result<()> { + let acct = self.accounts.entry(address).or_default(); + f(acct.balance.get_or_insert(Default::default())); + Ok(()) + } + fn update_nonce(&mut self, address: Address, f: impl FnOnce(&mut U256)) -> anyhow::Result<()> { + let acct = self.accounts.entry(address).or_default(); + f(acct.nonce.get_or_insert(Default::default())); + Ok(()) + } + fn set_code(&mut self, address: Address, code: Either<&[u8], H256>) -> anyhow::Result<()> { + let acct = self.accounts.entry(address).or_default(); + match code { + Either::Left(bytes) => { + acct.code = Some(keccak_hash::keccak(bytes).into_uint()); + acct.code_length = Some(U256::from(bytes.len())) + } + Either::Right(hash) => acct.code = Some(hash.into_uint()), + }; + Ok(()) + } + fn reporting_destroy(&mut self, address: Address) -> anyhow::Result> { + self.accounts.remove(&address); + Ok(None) + } + fn create_storage(&mut self, address: Address) -> anyhow::Result<()> { + let _ = address; + Ok(()) + } + fn destroy_storage(&mut self, address: Address) -> anyhow::Result<()> { + self.accounts + .entry(address) + .and_modify(|it| it.storage.clear()); + Ok(()) + } + fn store_int(&mut self, address: Address, slot: U256, value: U256) -> anyhow::Result<()> { + self.accounts + .entry(address) + .or_default() + .storage + .insert(slot, value); + Ok(()) + } + fn store_hash(&mut self, address: Address, hash: H256, value: H256) -> anyhow::Result<()> { + self.accounts + .entry(address) + .or_default() + .storage + .insert(hash.into_uint(), value.into_uint()); + Ok(()) + } + fn load_int(&mut self, address: Address, slot: U256) -> anyhow::Result { + Ok(self + .accounts + .get(&address) + .context("no account")? + .storage + .get(&slot) + .copied() + .unwrap_or_default()) + } + fn reporting_destroy_slot( + &mut self, + address: Address, + slot: U256, + ) -> anyhow::Result> { + self.accounts.entry(address).and_modify(|it| { + it.storage.remove(&slot); + }); + Ok(None) + } + fn mask_storage(&mut self, masks: BTreeMap>) -> anyhow::Result<()> { + let _ = masks; + Ok(()) + } + fn mask(&mut self, paths: impl IntoIterator) -> anyhow::Result<()> { + let _ = paths; + Ok(()) + } + fn root(&mut self) -> H256 { + let mut it = [0; 32]; + smt_trie::utils::hashout2u(self.as_smt().root).to_big_endian(&mut it); + H256(it) + } +} + +// Having optional fields here is an odd decision, +// but without the distinction, +// the wire tests fail. +// This may be a bug in the SMT library. +#[derive(Default, Clone, Debug)] +pub struct Type2Entry { + pub balance: Option, + pub nonce: Option, + pub code: Option, + pub code_length: Option, + pub storage: BTreeMap, +} + +// This is a buffered version +#[derive(Clone, Debug)] +pub struct Type2World { + accounts: BTreeMap, + hashed_out: BTreeMap, +} + +impl Type2World { + /// # Panics + /// - On untrusted inputs: . + pub fn as_smt(&self) -> smt_trie::smt::Smt { + let mut smt = smt_trie::smt::Smt::::default(); + + for (key, hash) in &self.hashed_out { + smt.set_hash( + key.into_smt_bits(), + smt_trie::smt::HashOut { + elements: { + let ethereum_types::U256(arr) = hash.into_uint(); + arr.map(plonky2::field::goldilocks_field::GoldilocksField) + }, + }, + ); + } + for ( + addr, + Type2Entry { + balance, + nonce, + code, + code_length, + storage, + }, + ) in self.accounts.iter() + { + use smt_trie::keys::{key_balance, key_code, key_code_length, key_nonce, key_storage}; + + for (value, key_fn) in [ + (balance, key_balance as fn(_) -> _), + (nonce, key_nonce), + (code, key_code), + (code_length, key_code_length), + ] { + if let Some(value) = value { + smt.set(key_fn(*addr), *value); + } + } + for (slot, value) in storage { + smt.set(key_storage(*addr, *slot), *value); + } + } + smt + } + + pub fn new_unchecked( + accounts: BTreeMap, + hashed_out: BTreeMap, + ) -> Self { + Self { + accounts, + hashed_out, + } + } +} diff --git a/zero/src/bin/trie_diff.rs b/zero/src/bin/trie_diff.rs index c211cc528..e580684d2 100644 --- a/zero/src/bin/trie_diff.rs +++ b/zero/src/bin/trie_diff.rs @@ -147,8 +147,8 @@ async fn main() -> Result<()> { &DebugOutputTries { state_trie: observer.data[prover_tries.batch_index] .tries - .state - .as_hashed_partial_trie() + .world + .state_trie() .clone(), transaction_trie: observer.data[prover_tries.batch_index] .tries From 1e1e76275ae79a931ae0c13e595065ff5f72e148 Mon Sep 17 00:00:00 2001 From: Serge Radinovich <47865535+sergerad@users.noreply.github.com> Date: Tue, 29 Oct 2024 01:40:02 +1300 Subject: [PATCH 11/17] BlockInterval support for hash ranges (#728) * Refactor BlockInterval::new() to facilitate hash range inputs * Update scripts * Remove blockid from interval * Script and into() cleanup * comments and cleanup * check end vs start in range * fix script call in workflow * fix checkpoint arg * add trait and mocks * replace all cached provider * cleanup tests * lint / cleanup * pr fixes * fix mock macro only tests and log proof count * rm block check and fix sh/readme changes * fix taplo * devdep * move provider * generalize BlockIntervalProvider impl * partial fix * compiling * fix provider generics * reinstate unit tests * add comment * add block to errors * rm expect * add where clause --- .github/workflows/jerigon-native.yml | 4 +- .github/workflows/jerigon-zero.yml | 33 +-- Cargo.lock | 71 +++++ scripts/prove_rpc.sh | 116 ++++---- zero/Cargo.toml | 4 +- zero/README.md | 10 +- zero/src/bin/leader.rs | 32 ++- zero/src/bin/leader/cli.rs | 19 +- zero/src/bin/leader/client.rs | 36 +-- zero/src/bin/rpc.rs | 18 +- zero/src/block_interval.rs | 387 +++++++++++++++++---------- zero/src/parsing.rs | 18 -- zero/src/provider.rs | 17 +- zero/src/rpc/mod.rs | 19 +- zero/src/rpc/native/mod.rs | 3 +- zero/src/rpc/native/state.rs | 1 + 16 files changed, 486 insertions(+), 302 deletions(-) diff --git a/.github/workflows/jerigon-native.yml b/.github/workflows/jerigon-native.yml index 29a380c3a..1ef443d9d 100644 --- a/.github/workflows/jerigon-native.yml +++ b/.github/workflows/jerigon-native.yml @@ -74,14 +74,14 @@ jobs: run: | ETH_RPC_URL="$(kurtosis port print cancun-testnet el-2-erigon-lighthouse ws-rpc)" ulimit -n 8192 - OUTPUT_TO_TERMINAL=true ./scripts/prove_rpc.sh 0x1 0xf $ETH_RPC_URL native true 3000 100 test_only + OUTPUT_TO_TERMINAL=true ./scripts/prove_rpc.sh 1 15 $ETH_RPC_URL native 0 3000 100 test_only echo "Proving blocks in test_only mode finished" - name: Run prove blocks with native tracer in real mode run: | ETH_RPC_URL="$(kurtosis port print cancun-testnet el-2-erigon-lighthouse ws-rpc)" rm -rf proofs/* circuits/* ./proofs.json test.out verify.out leader.out - OUTPUT_TO_TERMINAL=true RUN_VERIFICATION=true ./scripts/prove_rpc.sh 0x4 0x7 $ETH_RPC_URL native true 3000 100 + OUTPUT_TO_TERMINAL=true RUN_VERIFICATION=true ./scripts/prove_rpc.sh 4 7 $ETH_RPC_URL native 3 3000 100 echo "Proving blocks in real mode finished" - name: Shut down network diff --git a/.github/workflows/jerigon-zero.yml b/.github/workflows/jerigon-zero.yml index 216b32f8c..e0dc4c40f 100644 --- a/.github/workflows/jerigon-zero.yml +++ b/.github/workflows/jerigon-zero.yml @@ -10,7 +10,6 @@ on: branches: - "**" - env: CARGO_TERM_COLOR: always REGISTRY: ghcr.io @@ -26,16 +25,16 @@ jobs: uses: actions/checkout@v4 - name: Checkout test-jerigon-network sources - uses: actions/checkout@v4 + uses: actions/checkout@v4 with: repository: 0xPolygonZero/jerigon-test-network - ref: 'feat/kurtosis-network' + ref: "feat/kurtosis-network" path: jerigon-test-network - uses: actions-rust-lang/setup-rust-toolchain@v1 - + - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v3 - name: Login to GitHub Container Registry uses: docker/login-action@v2 @@ -57,39 +56,35 @@ jobs: #It is much easier to use cast tool in scripts so install foundry - name: Install Foundry - uses: foundry-rs/foundry-toolchain@v1 + uses: foundry-rs/foundry-toolchain@v1 - name: Run cancun test network run: | docker pull ghcr.io/0xpolygonzero/erigon:feat-zero - kurtosis run --enclave cancun-testnet github.com/ethpandaops/ethereum-package@4.0.0 --args-file jerigon-test-network/network_params.yml + kurtosis run --enclave cancun-testnet github.com/ethpandaops/ethereum-package@4.0.0 --args-file jerigon-test-network/network_params.yml - name: Generate blocks with transactions run: | - ETH_RPC_URL="$(kurtosis port print cancun-testnet el-2-erigon-lighthouse ws-rpc)" - cast rpc eth_blockNumber --rpc-url $ETH_RPC_URL - cd jerigon-test-network && set -a && source .env && set +a - bash ./tests/generate_transactions.sh - + ETH_RPC_URL="$(kurtosis port print cancun-testnet el-2-erigon-lighthouse ws-rpc)" + cast rpc eth_blockNumber --rpc-url $ETH_RPC_URL + cd jerigon-test-network && set -a && source .env && set +a + bash ./tests/generate_transactions.sh + - name: Run prove blocks with zero tracer in test_only mode run: | ETH_RPC_URL="$(kurtosis port print cancun-testnet el-2-erigon-lighthouse ws-rpc)" ulimit -n 8192 - OUTPUT_TO_TERMINAL=true ./scripts/prove_rpc.sh 0x1 0xf $ETH_RPC_URL jerigon true 3000 100 test_only + OUTPUT_TO_TERMINAL=true ./scripts/prove_rpc.sh 1 15 $ETH_RPC_URL jerigon 0 3000 100 test_only echo "Proving blocks in test_only mode finished" - - name: Run prove blocks with zero tracer in real mode run: | ETH_RPC_URL="$(kurtosis port print cancun-testnet el-2-erigon-lighthouse ws-rpc)" rm -rf proofs/* circuits/* ./proofs.json test.out verify.out leader.out - OUTPUT_TO_TERMINAL=true RUN_VERIFICATION=true ./scripts/prove_rpc.sh 0x2 0x5 $ETH_RPC_URL jerigon true 3000 100 + OUTPUT_TO_TERMINAL=true RUN_VERIFICATION=true ./scripts/prove_rpc.sh 2 5 $ETH_RPC_URL jerigon 1 3000 100 echo "Proving blocks in real mode finished" - + - name: Shut down network run: | kurtosis enclave rm -f cancun-testnet kurtosis engine stop - - - diff --git a/Cargo.lock b/Cargo.lock index 06c4aa6a1..0dc5ace4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1825,6 +1825,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + [[package]] name = "dunce" version = "1.0.5" @@ -2180,6 +2186,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + [[package]] name = "funty" version = "2.0.0" @@ -3048,6 +3060,32 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mockall" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c28b3fb6d753d28c20e826cd46ee611fda1cf3cde03a443a974043247c065a" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "341014e7f530314e9a1fdbc7400b244efea7122662c96bfa248c31da5bfb2020" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "mpt_trie" version = "0.4.1" @@ -3780,6 +3818,32 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "predicates" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9086cc7640c29a356d1a29fd134380bee9d8f79a17410aa76e7ad295f42c97" +dependencies = [ + "anstyle", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931" + +[[package]] +name = "predicates-tree" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "pretty_env_logger" version = "0.5.0" @@ -4806,6 +4870,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + [[package]] name = "text-size" version = "1.1.1" @@ -5814,6 +5884,7 @@ dependencies = [ "jemallocator", "keccak-hash 0.10.0", "lru", + "mockall", "mpt_trie", "num-traits", "once_cell", diff --git a/scripts/prove_rpc.sh b/scripts/prove_rpc.sh index 49848fdfe..d56d202fb 100755 --- a/scripts/prove_rpc.sh +++ b/scripts/prove_rpc.sh @@ -1,11 +1,11 @@ #!/bin/bash # Args: -# 1 --> Start block idx -# 2 --> End block index (inclusive) +# 1 --> Start block (number in decimal or block hash with prefix 0x). E.g. `1234` or `0x1d5e7a08dd1f4ce7fa52afe7f4960d78e82e508c874838dee594d5300b8df625`. +# 2 --> End block (number or hash, inclusive). Same format as start block. # 3 --> Rpc endpoint:port (eg. http://35.246.1.96:8545) # 4 --> Rpc type (eg. jerigon / native) -# 5 --> Ignore previous proofs (boolean) +# 5 --> Checkpoint block (number or hash). If argument is missing, start block predecessor will be used. # 6 --> Backoff in milliseconds (optional [default: 0]) # 7 --> Number of retries (optional [default: 0]) # 8 --> Test run only flag `test_only` (optional) @@ -38,13 +38,12 @@ REPO_ROOT=$(git rev-parse --show-toplevel) PROOF_OUTPUT_DIR="${REPO_ROOT}/proofs" OUT_LOG_PATH="${PROOF_OUTPUT_DIR}/b$1_$2.log" ALWAYS_WRITE_LOGS=0 # Change this to `1` if you always want logs to be written. -TOT_BLOCKS=$(($2-$1+1)) START_BLOCK=$1 END_BLOCK=$2 NODE_RPC_URL=$3 NODE_RPC_TYPE=$4 -IGNORE_PREVIOUS_PROOFS=$5 +CHECKPOINT_BLOCK=$5 BACKOFF=${6:-0} RETRIES=${7:-0} @@ -56,41 +55,28 @@ RUN_VERIFICATION="${RUN_VERIFICATION:-false}" # Recommended soft file handle limit. Will warn if it is set lower. RECOMMENDED_FILE_HANDLE_LIMIT=8192 -mkdir -p $PROOF_OUTPUT_DIR +mkdir -p "$PROOF_OUTPUT_DIR" -if $IGNORE_PREVIOUS_PROOFS ; then - # Set checkpoint height to previous block number for the first block in range - prev_proof_num=$(($1-1)) - PREV_PROOF_EXTRA_ARG="--checkpoint-block-number ${prev_proof_num}" +# Handle checkpoint block arg +if [ -n "$CHECKPOINT_BLOCK" ] ; then + # Checkpoint block provided, pass it to the prover as a flag + PREV_PROOF_EXTRA_ARG="--checkpoint-block $CHECKPOINT_BLOCK" else + # Checkpoint block not provided, but is required for hash-based start block + if [[ $START_BLOCK == 0x* ]]; then + echo "Checkpoint block is required when specifying blocks by hash" + exit 1 + fi + + # Checkpoint block not provided, deduce proof starting point from the start block if [[ $1 -gt 1 ]]; then prev_proof_num=$(($1-1)) PREV_PROOF_EXTRA_ARG="-f ${PROOF_OUTPUT_DIR}/b${prev_proof_num}.zkproof" fi fi -# Convert hex to decimal parameters -if [[ $START_BLOCK == 0x* ]]; then - START_BLOCK=$((16#${START_BLOCK#"0x"})) -fi -if [[ $END_BLOCK == 0x* ]]; then - END_BLOCK=$((16#${END_BLOCK#"0x"})) -fi - -# Define block interval -if [ $END_BLOCK == '-' ]; then - # Follow from the start block to the end of the chain - BLOCK_INTERVAL=$START_BLOCK.. -elif [ $START_BLOCK == $END_BLOCK ]; then - # Single block - BLOCK_INTERVAL=$START_BLOCK -else - # Block range - BLOCK_INTERVAL=$START_BLOCK..=$END_BLOCK -fi - # Print out a warning if the we're using `native` and our file descriptor limit is too low. Don't bother if we can't find `ulimit`. -if [ $(command -v ulimit) ] && [ $NODE_RPC_TYPE == "native" ] +if [ "$(command -v ulimit)" ] && [ "$NODE_RPC_TYPE" == "native" ] then file_desc_limit=$(ulimit -n) @@ -108,49 +94,77 @@ fi # other non-proving code. if [[ $8 == "test_only" ]]; then # test only run - echo "Proving blocks ${BLOCK_INTERVAL} in a test_only mode now... (Total: ${TOT_BLOCKS})" - command='cargo r --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --proof-output-dir $PROOF_OUTPUT_DIR --block-batch-size $BLOCK_BATCH_SIZE rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$NODE_RPC_URL" --block-interval $BLOCK_INTERVAL $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' + echo "Proving blocks from ($START_BLOCK) to ($END_BLOCK)" + command="cargo r --release --package zero --bin leader -- \ +--test-only \ +--runtime in-memory \ +--load-strategy on-demand \ +--proof-output-dir $PROOF_OUTPUT_DIR \ +--block-batch-size $BLOCK_BATCH_SIZE \ +rpc \ +--rpc-type $NODE_RPC_TYPE \ +--rpc-url $NODE_RPC_URL \ +--start-block $START_BLOCK \ +--end-block $END_BLOCK \ +--backoff $BACKOFF \ +--max-retries $RETRIES \ +$PREV_PROOF_EXTRA_ARG" + if [ "$OUTPUT_TO_TERMINAL" = true ]; then - eval $command + eval "$command" retVal=$? echo -e "Proof witness generation finished with result: $retVal" exit $retVal else - eval $command > $OUT_LOG_PATH 2>&1 - if grep -q 'All proof witnesses have been generated successfully.' $OUT_LOG_PATH; then + eval "$command" > "$OUT_LOG_PATH" 2>&1 + if grep -q 'All proof witnesses have been generated successfully.' "$OUT_LOG_PATH"; then echo -e "Success - Note this was just a test, not a proof" # Remove the log on success if we don't want to keep it. if [ $ALWAYS_WRITE_LOGS -ne 1 ]; then - rm $OUT_LOG_PATH + rm "$OUT_LOG_PATH" fi exit else - echo "Failed to create proof witnesses. See ${OUT_LOG_PATH} for more details." + echo "Failed to create proof witnesses. See $OUT_LOG_PATH for more details." exit 1 fi fi else # normal run - echo "Proving blocks ${BLOCK_INTERVAL} now... (Total: ${TOT_BLOCKS})" - command='cargo r --release --package zero --bin leader -- --runtime in-memory --load-strategy on-demand --proof-output-dir $PROOF_OUTPUT_DIR --block-batch-size $BLOCK_BATCH_SIZE rpc --rpc-type "$NODE_RPC_TYPE" --rpc-url "$3" --block-interval $BLOCK_INTERVAL $PREV_PROOF_EXTRA_ARG --backoff "$BACKOFF" --max-retries "$RETRIES" ' + echo "Proving blocks from ($START_BLOCK) to ($END_BLOCK)" + command="cargo r --release --package zero --bin leader -- \ +--runtime in-memory \ +--load-strategy on-demand \ +--proof-output-dir $PROOF_OUTPUT_DIR \ +--block-batch-size $BLOCK_BATCH_SIZE \ +rpc \ +--rpc-type $NODE_RPC_TYPE \ +--rpc-url $3 \ +--start-block $START_BLOCK \ +--end-block $END_BLOCK \ +--backoff $BACKOFF \ +--max-retries $RETRIES \ +$PREV_PROOF_EXTRA_ARG " + if [ "$OUTPUT_TO_TERMINAL" = true ]; then - eval $command + eval "$command" echo -e "Proof generation finished with result: $?" else - eval $command > $OUT_LOG_PATH 2>&1 + eval "$command" > "$OUT_LOG_PATH" 2>&1 retVal=$? if [ $retVal -ne 0 ]; then # Some error occurred, display the logs and exit. - cat $OUT_LOG_PATH - echo "Block ${i} errored. See ${OUT_LOG_PATH} for more details." + cat "$OUT_LOG_PATH" + echo "Error occurred. See $OUT_LOG_PATH for more details." exit $retVal else # Remove the log on success if we don't want to keep it. if [ $ALWAYS_WRITE_LOGS -ne 1 ]; then - rm $OUT_LOG_PATH + rm "$OUT_LOG_PATH" fi fi - echo "Successfully generated ${TOT_BLOCKS} proofs!" + proof_count=$(grep -c 'INFO zero::prover: Proving block \d' < "$OUT_LOG_PATH") + echo "Successfully generated $proof_count proofs!" fi fi @@ -160,15 +174,15 @@ if [ "$RUN_VERIFICATION" = true ]; then echo "Running the verification for the last proof..." proof_file_name=$PROOF_OUTPUT_DIR/b$END_BLOCK.zkproof - echo "Verifying the proof of the latest block in the interval:" $proof_file_name - cargo r --release --package zero --bin verifier -- -f $proof_file_name > $PROOF_OUTPUT_DIR/verify.out 2>&1 + echo "Verifying the proof of the latest block in the interval:" "$proof_file_name" + cargo r --release --package zero --bin verifier -- -f "$proof_file_name" > "$PROOF_OUTPUT_DIR/verify.out" 2>&1 - if grep -q 'All proofs verified successfully!' $PROOF_OUTPUT_DIR/verify.out; then + if grep -q 'All proofs verified successfully!' "$PROOF_OUTPUT_DIR/verify.out"; then echo "$proof_file_name verified successfully!"; - rm $PROOF_OUTPUT_DIR/verify.out + rm "$PROOF_OUTPUT_DIR/verify.out" else # Some error occurred with verification, display the logs and exit. - cat $PROOF_OUTPUT_DIR/verify.out + cat "$PROOF_OUTPUT_DIR/verify.out" echo "There was an issue with proof verification. See $PROOF_OUTPUT_DIR/verify.out for more details."; exit 1 fi diff --git a/zero/Cargo.toml b/zero/Cargo.toml index 7cbf2f351..88c8c580b 100644 --- a/zero/Cargo.toml +++ b/zero/Cargo.toml @@ -51,11 +51,13 @@ zk_evm_common.workspace = true [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.4" - [build-dependencies] anyhow.workspace = true vergen-git2 = { version = "1.0.0", features = ["build"] } +[dev-dependencies] +mockall = "0.13.0" + [features] default = ["eth_mainnet"] diff --git a/zero/README.md b/zero/README.md index 936a73c91..e320b8d62 100644 --- a/zero/README.md +++ b/zero/README.md @@ -425,13 +425,13 @@ For testing proof generation for blocks, the `testing` branch should be used. If you want to generate a full block proof, you can use `tools/prove_rpc.sh`: ```sh -./prove_rpc.sh +./prove_rpc.sh ``` Which may look like this: ```sh -./prove_rpc.sh 17 18 http://127.0.0.1:8545 jerigon false +./prove_rpc.sh 17 18 http://127.0.0.1:8545 jerigon ``` Which will attempt to generate proofs for blocks `17` & `18` consecutively and incorporate the previous block proof during generation. @@ -439,7 +439,7 @@ Which will attempt to generate proofs for blocks `17` & `18` consecutively and i A few other notes: - Proving blocks is very resource intensive in terms of both CPU and memory. You can also only generate the witness for a block instead (see [Generating Witnesses Only](#generating-witnesses-only)) to significantly reduce the CPU and memory requirements. -- Because incorporating the previous block proof requires a chain of proofs back to the last checkpoint height, you can also disable this requirement by passing `true` for `` (which internally just sets the current checkpoint height to the previous block height). +- Because incorporating the previous block proof requires a chain of proofs back to the last checkpoint height, you must specify a ``. The above example omits this argument which causes the command to treat block `16` as the checkpoint. - When proving multiple blocks concurrently, one may need to increase the system resource usage limit because of the number of RPC connections opened simultaneously, in particular when running a native tracer. For Linux systems, it is recommended to set `ulimit` to 8192. ### Generating Witnesses Only @@ -447,13 +447,13 @@ A few other notes: If you want to test a block without the high CPU & memory requirements that come with creating a full proof, you can instead generate only the witness using `tools/prove_rpc.sh` in the `test_only` mode: ```sh -./prove_rpc.sh test_only +./prove_rpc.sh test_only ``` Filled in: ```sh -./prove_rpc.sh 18299898 18299899 http://34.89.57.138:8545 jerigon true 0 0 test_only +./prove_rpc.sh 18299898 18299899 http://34.89.57.138:8545 jerigon 18299897 0 0 test_only ``` Finally, note that both of these testing scripts force proof generation to be sequential by allowing only one worker. Because of this, this is not a realistic representation of performance but makes the debugging logs much easier to follow. diff --git a/zero/src/bin/leader.rs b/zero/src/bin/leader.rs index 6daaf599f..5d11845c6 100644 --- a/zero/src/bin/leader.rs +++ b/zero/src/bin/leader.rs @@ -5,12 +5,12 @@ use std::sync::Arc; use anyhow::Result; use clap::Parser; use cli::Command; -use client::RpcParams; use paladin::config::Config; use paladin::runtime::Runtime; use tracing::info; use zero::env::load_dotenvy_vars_if_present; use zero::prover::{ProofRuntime, ProverConfig}; +use zero::rpc::retry::build_http_retry_provider; use zero::{ block_interval::BlockInterval, prover_state::persistence::set_circuit_cache_dir_env_if_not_set, }; @@ -103,26 +103,36 @@ async fn main() -> Result<()> { Command::Rpc { rpc_url, rpc_type, - block_interval, - checkpoint_block_number, + checkpoint_block, previous_proof, block_time, + start_block, + end_block, backoff, max_retries, } => { + // Construct the provider. let previous_proof = get_previous_proof(previous_proof)?; - let block_interval = BlockInterval::new(&block_interval)?; + let retry_provider = build_http_retry_provider(rpc_url.clone(), backoff, max_retries)?; + let cached_provider = Arc::new(zero::provider::CachedProvider::new( + retry_provider, + rpc_type, + )); + // Construct the block interval. + let block_interval = + BlockInterval::new(cached_provider.clone(), start_block, end_block).await?; + + // Convert the checkpoint block to a block number. + let checkpoint_block_number = + BlockInterval::block_to_num(cached_provider.clone(), checkpoint_block).await?; + + // Prove the block interval. info!("Proving interval {block_interval}"); client_main( proof_runtime, - RpcParams { - rpc_url, - rpc_type, - backoff, - max_retries, - block_time, - }, + cached_provider, + block_time, block_interval, LeaderConfig { checkpoint_block_number, diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index ad9270ee8..c085ae83f 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use alloy::eips::BlockId; use alloy::transports::http::reqwest::Url; use clap::{Parser, Subcommand, ValueEnum, ValueHint}; use zero::prover::cli::CliProverConfig; @@ -45,6 +46,7 @@ pub enum WorkerRunMode { Default, } +#[allow(clippy::large_enum_variant)] #[derive(Subcommand)] pub(crate) enum Command { /// Deletes all the previously cached circuits. @@ -63,12 +65,17 @@ pub(crate) enum Command { // The node RPC type (jerigon / native). #[arg(long, short = 't', default_value = "jerigon")] rpc_type: RpcType, - /// The block interval for which to generate a proof. - #[arg(long, short = 'i')] - block_interval: String, - /// The checkpoint block number. - #[arg(short, long, default_value_t = 0)] - checkpoint_block_number: u64, + /// The start of the block range to prove (inclusive). + #[arg(long, short = 's')] + start_block: BlockId, + /// The end of the block range to prove (inclusive). + /// If not provided, leader will work in dynamic mode from `start_block` + /// following head of the blockchain. + #[arg(long, short = 'e')] + end_block: Option, + /// The checkpoint block. + #[arg(short, long, default_value = "0")] + checkpoint_block: BlockId, /// The previous proof output. #[arg(long, short = 'f', value_hint = ValueHint::FilePath)] previous_proof: Option, diff --git a/zero/src/bin/leader/client.rs b/zero/src/bin/leader/client.rs index 343a2cdcb..6f2015833 100644 --- a/zero/src/bin/leader/client.rs +++ b/zero/src/bin/leader/client.rs @@ -1,7 +1,8 @@ use std::sync::Arc; +use alloy::providers::Provider; use alloy::rpc::types::{BlockId, BlockNumberOrTag}; -use alloy::transports::http::reqwest::Url; +use alloy::transports::Transport; use anyhow::{anyhow, Result}; use tokio::sync::mpsc; use tracing::info; @@ -9,20 +10,11 @@ use zero::block_interval::{BlockInterval, BlockIntervalStream}; use zero::pre_checks::check_previous_proof_and_checkpoint; use zero::proof_types::GeneratedBlockProof; use zero::prover::{self, BlockProverInput, ProverConfig}; +use zero::provider::CachedProvider; use zero::rpc; -use zero::rpc::{retry::build_http_retry_provider, RpcType}; use crate::ProofRuntime; -#[derive(Debug)] -pub struct RpcParams { - pub rpc_url: Url, - pub rpc_type: RpcType, - pub backoff: u64, - pub max_retries: u32, - pub block_time: u64, -} - #[derive(Debug)] pub struct LeaderConfig { pub checkpoint_block_number: u64, @@ -31,24 +23,21 @@ pub struct LeaderConfig { } /// The main function for the client. -pub(crate) async fn client_main( +pub(crate) async fn client_main( proof_runtime: Arc, - rpc_params: RpcParams, + cached_provider: Arc>, + block_time: u64, block_interval: BlockInterval, mut leader_config: LeaderConfig, -) -> Result<()> { +) -> Result<()> +where + ProviderT: Provider + 'static, + TransportT: Transport + Clone, +{ use futures::StreamExt; let test_only = leader_config.prover_config.test_only; - let cached_provider = Arc::new(zero::provider::CachedProvider::new( - build_http_retry_provider( - rpc_params.rpc_url.clone(), - rpc_params.backoff, - rpc_params.max_retries, - )?, - )); - if !test_only { // For actual proof runs, perform a sanity check on the provided inputs. check_previous_proof_and_checkpoint( @@ -76,7 +65,7 @@ pub(crate) async fn client_main( let mut block_interval_stream: BlockIntervalStream = match block_interval { block_interval @ BlockInterval::FollowFrom { .. } => { block_interval - .into_unbounded_stream(cached_provider.clone(), rpc_params.block_time) + .into_unbounded_stream(cached_provider.clone(), block_time) .await? } _ => block_interval.into_bounded_stream()?, @@ -92,7 +81,6 @@ pub(crate) async fn client_main( cached_provider.clone(), block_id, leader_config.checkpoint_block_number, - rpc_params.rpc_type, ) .await?; block_tx diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index 164751df2..a8a42a6d4 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -25,7 +25,6 @@ struct FetchParams { pub start_block: u64, pub end_block: u64, pub checkpoint_block_number: Option, - pub rpc_type: RpcType, } #[derive(Args, Clone, Debug)] @@ -98,13 +97,9 @@ where let (block_num, _is_last_block) = block_interval_elem?; let block_id = BlockId::Number(BlockNumberOrTag::Number(block_num)); // Get the prover input for particular block. - let result = rpc::block_prover_input( - cached_provider.clone(), - block_id, - checkpoint_block_number, - params.rpc_type, - ) - .await?; + let result = + rpc::block_prover_input(cached_provider.clone(), block_id, checkpoint_block_number) + .await?; block_prover_inputs.push(result); } @@ -114,11 +109,12 @@ where impl Cli { /// Execute the cli command. pub async fn execute(self) -> anyhow::Result<()> { - let cached_provider = Arc::new(CachedProvider::new(build_http_retry_provider( + let retry_provider = build_http_retry_provider( self.config.rpc_url.clone(), self.config.backoff, self.config.max_retries, - )?)); + )?; + let cached_provider = Arc::new(CachedProvider::new(retry_provider, self.config.rpc_type)); match self.command { Command::Fetch { @@ -130,7 +126,6 @@ impl Cli { start_block, end_block, checkpoint_block_number, - rpc_type: self.config.rpc_type, }; let block_prover_inputs = @@ -156,7 +151,6 @@ impl Cli { start_block: block_number, end_block: block_number, checkpoint_block_number: None, - rpc_type: self.config.rpc_type, }; let block_prover_inputs = diff --git a/zero/src/block_interval.rs b/zero/src/block_interval.rs index e424076e0..27d03fe3c 100644 --- a/zero/src/block_interval.rs +++ b/zero/src/block_interval.rs @@ -1,16 +1,52 @@ use std::pin::Pin; use std::sync::Arc; +use std::{future::Future, ops::Range}; -use alloy::primitives::B256; -use alloy::rpc::types::eth::BlockId; -use alloy::{hex, providers::Provider, transports::Transport}; +use alloy::rpc::types::{eth::BlockId, Block}; use anyhow::{anyhow, Result}; use async_stream::try_stream; use futures::Stream; +#[cfg(test)] +use mockall::automock; use tracing::info; -use crate::parsing; -use crate::provider::CachedProvider; +#[cfg_attr(test, automock)] +pub trait BlockIntervalProvider { + fn get_block_by_id( + &self, + block_id: BlockId, + ) -> impl Future>> + Send; + + fn latest_block_number(&self) -> impl Future> + Send; +} + +#[cfg(not(test))] +mod block_interval_provider_impl { + use alloy::providers::Provider; + use alloy::rpc::types::BlockTransactionsKind; + use alloy::transports::Transport; + + use super::{Block, BlockId, BlockIntervalProvider}; + + /// Implements the [`BlockIntervalProvider`] trait for [`Provider`]. + impl BlockIntervalProvider for P + where + T: Transport + Clone, + P: Provider, + { + /// Retrieves block without transaction contents from the provider. + async fn get_block_by_id(&self, block_id: BlockId) -> anyhow::Result> { + Ok(self + .get_block(block_id, BlockTransactionsKind::Hashes) + .await?) + } + + /// Retrieves the latest block number from the provider. + async fn latest_block_number(&self) -> anyhow::Result { + Ok(self.get_block_number().await?) + } + } +} /// The async stream of block numbers. /// The second bool flag indicates if the element is last in the interval. @@ -20,9 +56,9 @@ pub type BlockIntervalStream = Pin), + Range(Range), // Dynamic interval from the start block to the latest network block FollowFrom { // Interval starting block number @@ -31,64 +67,43 @@ pub enum BlockInterval { } impl BlockInterval { - /// Create a new block interval + /// Creates a new block interval. /// - /// A valid block range is of the form: - /// * `block_number` for a single block number - /// * `lhs..rhs`, `lhs..=rhs` as an exclusive/inclusive range - /// * `lhs..` for a range starting from `lhs` to the chain tip. `lhs..=` - /// is also valid format. + /// If end_block is None, the interval is unbounded and will follow from + /// start_block. If start_block == end_block, the interval is a single + /// block. Otherwise, the interval is an inclusive range from start_block to + /// end_block. /// - /// # Example - /// - /// ```rust - /// # use alloy::rpc::types::eth::BlockId; - /// # use zero::block_interval::BlockInterval; - /// assert_eq!(BlockInterval::new("0..10").unwrap(), BlockInterval::Range(0..10)); - /// assert_eq!(BlockInterval::new("0..=10").unwrap(), BlockInterval::Range(0..11)); - /// assert_eq!(BlockInterval::new("32141").unwrap(), BlockInterval::SingleBlockId(BlockId::Number(32141.into()))); - /// assert_eq!(BlockInterval::new("100..").unwrap(), BlockInterval::FollowFrom{start_block: 100}); - /// ``` - pub fn new(s: &str) -> anyhow::Result { - if (s.starts_with("0x") && s.len() == 66) || s.len() == 64 { - // Try to parse hash - let hash = s - .parse::() - .map_err(|_| anyhow!("invalid block hash '{s}'"))?; - return Ok(BlockInterval::SingleBlockId(BlockId::Hash(hash.into()))); - } - - // First we parse for inclusive range and then for exclusive range, - // because both separators start with `..` - if let Ok(range) = parsing::parse_range_inclusive(s) { - Ok(BlockInterval::Range(range)) - } else if let Ok(range) = parsing::parse_range_exclusive(s) { - Ok(BlockInterval::Range(range)) - } - // Now we look for the follow from range - else if s.contains("..") { - let mut split = s.trim().split("..").filter(|s| *s != "=" && !s.is_empty()); + /// end_block is always treated as inclusive because it may have been + /// specified as a block hash. + pub async fn new( + provider: Arc>, + start_block: BlockId, + end_block: Option, + ) -> Result { + // Ensure the start block is a valid block number. + let start_block_num = Self::block_to_num(provider.clone(), start_block).await?; - // Any other character after `..` or `..=` is invalid - if split.clone().count() > 1 { - return Err(anyhow!("invalid block interval range '{s}'")); + // Create the block interval. + match end_block { + // Start and end are the same. + Some(end_block) if end_block == start_block => { + Ok(BlockInterval::SingleBlockId(start_block_num)) } - let num = split - .next() - .map(|num| { - num.parse::() - .map_err(|_| anyhow!("invalid block number '{num}'")) - }) - .ok_or(anyhow!("invalid block interval range '{s}'"))??; - return Ok(BlockInterval::FollowFrom { start_block: num }); - } - // Only single block number is left to try to parse - else { - let num: u64 = s - .trim() - .parse() - .map_err(|_| anyhow!("invalid block interval range '{s}'"))?; - return Ok(BlockInterval::SingleBlockId(BlockId::Number(num.into()))); + // Bounded range provided. + Some(end_block) => { + let end_block_num = Self::block_to_num(provider.clone(), end_block).await?; + if end_block_num <= start_block_num { + return Err(anyhow!( + "invalid block interval range ({start_block_num}..{end_block_num})" + )); + } + Ok(BlockInterval::Range(start_block_num..end_block_num + 1)) + } + // Unbounded range provided. + None => Ok(BlockInterval::FollowFrom { + start_block: start_block_num, + }), } } @@ -96,10 +111,7 @@ impl BlockInterval { /// second bool flag indicates if the element is last in the interval. pub fn into_bounded_stream(self) -> Result { match self { - BlockInterval::SingleBlockId(BlockId::Number(num)) => { - let num = num - .as_number() - .ok_or(anyhow!("invalid block number '{num}'"))?; + BlockInterval::SingleBlockId(num) => { let range = (num..num + 1).map(|it| Ok((it, true))).collect::>(); Ok(Box::pin(futures::stream::iter(range))) @@ -110,42 +122,33 @@ impl BlockInterval { range.last_mut().map(|it| it.as_mut().map(|it| it.1 = true)); Ok(Box::pin(futures::stream::iter(range))) } - _ => Err(anyhow!( + BlockInterval::FollowFrom { .. } => Err(anyhow!( "could not create bounded stream from unbounded follow-from interval", )), } } + /// Returns the start block number of the interval. pub fn get_start_block(&self) -> Result { match self { - BlockInterval::SingleBlockId(BlockId::Number(num)) => { - let num_value = num - .as_number() - .ok_or_else(|| anyhow!("invalid block number '{num}'"))?; - Ok(num_value) // Return the valid block number - } + BlockInterval::SingleBlockId(num) => Ok(*num), BlockInterval::Range(range) => Ok(range.start), BlockInterval::FollowFrom { start_block, .. } => Ok(*start_block), - _ => Err(anyhow!("Unknown BlockInterval variant")), // Handle unknown variants } } /// Convert the block interval into an unbounded async stream of block /// numbers. Query the blockchain node for the latest block number. - pub async fn into_unbounded_stream( + pub async fn into_unbounded_stream( self, - cached_provider: Arc>, + provider: Arc + 'static>, block_time: u64, - ) -> Result - where - ProviderT: Provider + 'static, - TransportT: Transport + Clone, - { + ) -> Result { match self { BlockInterval::FollowFrom { start_block } => Ok(Box::pin(try_stream! { let mut current = start_block; loop { - let last_block_number = cached_provider.get_provider().await?.get_block_number().await.map_err(|e: alloy::transports::RpcError<_>| { + let last_block_number = provider.latest_block_number().await.map_err(|e| { anyhow!("could not retrieve latest block number from the provider: {e}") })?; @@ -166,15 +169,40 @@ impl BlockInterval { )), } } + + /// Converts a [`BlockId`] into a block number by querying the provider. + pub async fn block_to_num( + provider: Arc>, + block: BlockId, + ) -> Result { + let block_num = match block { + // Number already provided + BlockId::Number(num) => num + .as_number() + .ok_or_else(|| anyhow!("invalid block number '{num}'"))?, + + // Hash provided, query the provider for the block number. + BlockId::Hash(hash) => { + let block = provider + .get_block_by_id(BlockId::Hash(hash)) + .await + .map_err(|e| { + anyhow!("could not retrieve block number by hash from the provider: {e}") + })?; + block + .ok_or(anyhow!("block not found {hash}"))? + .header + .number + } + }; + Ok(block_num) + } } impl std::fmt::Display for BlockInterval { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - BlockInterval::SingleBlockId(block_id) => match block_id { - BlockId::Number(it) => f.write_fmt(format_args!("{}", it)), - BlockId::Hash(it) => f.write_fmt(format_args!("0x{}", &hex::encode(it.block_hash))), - }, + BlockInterval::SingleBlockId(num) => f.write_fmt(format_args!("{}", num)), BlockInterval::Range(range) => { write!(f, "{}..{}", range.start, range.end) } @@ -185,92 +213,174 @@ impl std::fmt::Display for BlockInterval { } } -impl std::str::FromStr for BlockInterval { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - BlockInterval::new(s) - } -} - #[cfg(test)] mod test { use alloy::primitives::B256; + use alloy::rpc::types::{Block, Header, Transaction}; + use alloy::transports::BoxTransport; + use mockall::predicate::*; + use MockBlockIntervalProvider; use super::*; - #[test] - fn can_create_block_interval_from_exclusive_range() { + type Mocker = MockBlockIntervalProvider; + + #[tokio::test] + async fn can_create_block_interval_from_inclusive_range() { assert_eq!( - BlockInterval::new("0..10").unwrap(), - BlockInterval::Range(0..10) + BlockInterval::new( + Arc::new(Mocker::new()), + BlockId::from(0), + Some(BlockId::from(10)) + ) + .await + .unwrap(), + BlockInterval::Range(0..11) ); } - #[test] - fn can_create_block_interval_from_inclusive_range() { + #[tokio::test] + async fn can_create_follow_from_block_interval() { assert_eq!( - BlockInterval::new("0..=10").unwrap(), - BlockInterval::Range(0..11) + BlockInterval::new(Arc::new(Mocker::new()), BlockId::from(100), None) + .await + .unwrap(), + BlockInterval::FollowFrom { start_block: 100 } ); } - #[test] - fn can_create_follow_from_block_interval() { + #[tokio::test] + async fn can_create_single_block_interval() { assert_eq!( - BlockInterval::new("100..").unwrap(), - BlockInterval::FollowFrom { start_block: 100 } + BlockInterval::new( + Arc::new(Mocker::new()), + BlockId::from(123415131), + Some(BlockId::from(123415131)) + ) + .await + .unwrap(), + BlockInterval::SingleBlockId(123415131) ); } - #[test] - fn can_create_single_block_interval() { + #[tokio::test] + async fn cannot_create_invalid_range() { assert_eq!( - BlockInterval::new("123415131").unwrap(), - BlockInterval::SingleBlockId(BlockId::Number(123415131.into())) + BlockInterval::new( + Arc::new(Mocker::new()), + BlockId::from(123415131), + Some(BlockId::from(0)) + ) + .await + .unwrap_err() + .to_string(), + anyhow!("invalid block interval range (123415131..0)").to_string() ); } - #[test] - fn new_interval_proper_single_block_error() { + #[tokio::test] + async fn can_create_single_block_interval_from_hash() { + // Mock the block for single block interval. + let mut mock = Mocker::new(); + let block_id = BlockId::Hash( + "0xb51ceca7ba912779ed6721d2b93849758af0d2354683170fb71dead6e439e6cb" + .parse::() + .unwrap() + .into(), + ); + mock_block(&mut mock, block_id, 12345); + + // Create the interval. + let mock = Arc::new(mock); assert_eq!( - BlockInterval::new("113A").err().unwrap().to_string(), - "invalid block interval range '113A'" + BlockInterval::new(mock, block_id, Some(block_id)) + .await + .unwrap(), + BlockInterval::SingleBlockId(12345) ); } - #[test] - fn new_interval_proper_range_error() { + #[tokio::test] + async fn can_create_block_interval_from_inclusive_hash_range() { + // Mock the blocks for the range. + let mut mock = Mocker::new(); + let start_block_id = BlockId::Hash( + "0xb51ceca7ba912779ed6721d2b93849758af0d2354683170fb71dead6e439e6cb" + .parse::() + .unwrap() + .into(), + ); + mock_block(&mut mock, start_block_id, 12345); + let end_block_id = BlockId::Hash( + "0x351ceca7ba912779ed6721d2b93849758af0d2354683170fb71dead6e439e6cb" + .parse::() + .unwrap() + .into(), + ); + mock_block(&mut mock, end_block_id, 12355); + + // Create the interval. + let mock = Arc::new(mock); assert_eq!( - BlockInterval::new("111...156").err().unwrap().to_string(), - "invalid block interval range '111...156'" + BlockInterval::new(mock, start_block_id, Some(end_block_id)) + .await + .unwrap(), + BlockInterval::Range(12345..12356) ); } - #[test] - fn new_interval_parse_block_hash() { + #[tokio::test] + async fn can_create_follow_from_block_interval_hash() { + // Mock a block for range to start from. + let start_block_id = BlockId::Hash( + "0xb51ceca7ba912779ed6721d2b93849758af0d2354683170fb71dead6e439e6cb" + .parse::() + .unwrap() + .into(), + ); + let mut mock = Mocker::new(); + mock_block(&mut mock, start_block_id, 12345); + + // Create the interval. + let mock = Arc::new(mock); assert_eq!( - BlockInterval::new( - "0xb51ceca7ba912779ed6721d2b93849758af0d2354683170fb71dead6e439e6cb" - ) - .unwrap(), - BlockInterval::SingleBlockId(BlockId::Hash( - "0xb51ceca7ba912779ed6721d2b93849758af0d2354683170fb71dead6e439e6cb" - .parse::() - .unwrap() - .into() - )) - ) + BlockInterval::new(mock, start_block_id, None) + .await + .unwrap(), + BlockInterval::FollowFrom { start_block: 12345 } + ); + } + + /// Configures the mock to expect a query for a block by id and return + /// the expected block number. + fn mock_block( + mock: &mut MockBlockIntervalProvider, + query_id: BlockId, + resulting_block_num: u64, + ) { + let mut block: Block = Block::default(); + block.header.number = resulting_block_num; + mock.expect_get_block_by_id() + .with(eq(query_id)) + .returning(move |_| { + let block = block.clone(); + Box::pin(async move { Ok(Some(block)) }) + }); } #[tokio::test] async fn can_into_bounded_stream() { use futures::StreamExt; let mut result = Vec::new(); - let mut stream = BlockInterval::new("1..10") - .unwrap() - .into_bounded_stream() - .unwrap(); + let mut stream = BlockInterval::new( + Arc::new(Mocker::new()), + BlockId::from(1), + Some(BlockId::from(9)), + ) + .await + .unwrap() + .into_bounded_stream() + .unwrap(); while let Some(val) = stream.next().await { result.push(val.unwrap()); } @@ -281,13 +391,4 @@ mod test { expected.last_mut().unwrap().1 = true; assert_eq!(result, expected); } - - #[test] - fn can_create_from_string() { - use std::str::FromStr; - assert_eq!( - &format!("{}", BlockInterval::from_str("0..10").unwrap()), - "0..10" - ); - } } diff --git a/zero/src/parsing.rs b/zero/src/parsing.rs index d1452a464..5643f82f5 100644 --- a/zero/src/parsing.rs +++ b/zero/src/parsing.rs @@ -34,19 +34,6 @@ where parse_range_gen(s, "..", false) } -/// Parse an inclusive range from a string. -/// -/// A valid range is of the form `lhs..=rhs`, where `lhs` and `rhs` are numbers. -pub(crate) fn parse_range_inclusive( - s: &str, -) -> Result, RangeParseError> -where - NumberT: Display + FromStr + From + Add, - NumberT::Err: Display, -{ - parse_range_gen(s, "..=", true) -} - pub(crate) fn parse_range_gen( s: &str, separator: SeparatorT, @@ -88,11 +75,6 @@ mod test { assert_eq!(parse_range_exclusive::("0..10"), Ok(0..10)); } - #[test] - fn it_parses_inclusive_ranges() { - assert_eq!(parse_range_inclusive::("0..=10"), Ok(0..11)); - } - #[test] fn it_handles_missing_lhs() { assert_eq!( diff --git a/zero/src/provider.rs b/zero/src/provider.rs index 876cb270c..a2168bbb7 100644 --- a/zero/src/provider.rs +++ b/zero/src/provider.rs @@ -2,14 +2,27 @@ use std::ops::{Deref, DerefMut}; use std::sync::Arc; use alloy::primitives::BlockHash; +use alloy::providers::RootProvider; use alloy::rpc::types::{Block, BlockId, BlockTransactionsKind}; use alloy::{providers::Provider, transports::Transport}; use anyhow::Context; use tokio::sync::{Mutex, Semaphore, SemaphorePermit}; +use crate::rpc::RpcType; + const CACHE_SIZE: usize = 1024; const MAX_NUMBER_OF_PARALLEL_REQUESTS: usize = 128; +impl Provider for CachedProvider +where + T: Transport + Clone, + P: Provider, +{ + fn root(&self) -> &RootProvider { + self.provider.root() + } +} + /// Wrapper around alloy provider to cache blocks and other /// frequently used data. pub struct CachedProvider { @@ -22,6 +35,7 @@ pub struct CachedProvider { blocks_by_number: Arc>>, blocks_by_hash: Arc>>, _phantom: std::marker::PhantomData, + pub rpc_type: RpcType, } pub struct ProviderGuard<'a, ProviderT> { @@ -48,7 +62,7 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - pub fn new(provider: ProviderT) -> Self { + pub fn new(provider: ProviderT, rpc_type: RpcType) -> Self { Self { provider: provider.into(), semaphore: Arc::new(Semaphore::new(MAX_NUMBER_OF_PARALLEL_REQUESTS)), @@ -58,6 +72,7 @@ where blocks_by_hash: Arc::new(Mutex::new(lru::LruCache::new( std::num::NonZero::new(CACHE_SIZE).unwrap(), ))), + rpc_type, _phantom: std::marker::PhantomData, } } diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index 007a4fdb2..016c1d242 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -45,13 +45,12 @@ pub async fn block_prover_input( cached_provider: Arc>, block_id: BlockId, checkpoint_block_number: u64, - rpc_type: RpcType, ) -> Result where ProviderT: Provider, TransportT: Transport + Clone, { - match rpc_type { + match cached_provider.rpc_type { RpcType::Jerigon => { jerigon::block_prover_input(cached_provider, block_id, checkpoint_block_number).await } @@ -105,8 +104,8 @@ where async move { let block = cached_provider .get_block((block_num as u64).into(), BlockTransactionsKind::Hashes) - .await - .context("couldn't get block")?; + .await? + .ok_or(anyhow!("block not found {block_num}"))?; anyhow::Ok([ (block.header.hash, Some(block_num)), (block.header.parent_hash, previous_block_number), @@ -212,8 +211,8 @@ where { let target_block = cached_provider .get_block(target_block_id, BlockTransactionsKind::Hashes) - .await?; - let target_block_number = target_block.header.number; + .await? + .ok_or(anyhow!("target block not found {}", target_block_id))?; let chain_id = cached_provider.get_provider().await?.get_chain_id().await?; // Grab interval checkpoint block state trie @@ -223,11 +222,15 @@ where BlockTransactionsKind::Hashes, ) .await? + .ok_or(anyhow!( + "checkpoint block not found {}", + checkpoint_block_number + ))? .header .state_root; let prev_hashes = - fetch_previous_block_hashes(cached_provider.clone(), target_block_number).await?; + fetch_previous_block_hashes(cached_provider.clone(), target_block.header.number).await?; let checkpoint_prev_hashes = fetch_previous_block_hashes(cached_provider, checkpoint_block_number + 1) // include the checkpoint block .await? @@ -238,7 +241,7 @@ where b_meta: BlockMetadata { block_beneficiary: target_block.header.miner.compat(), block_timestamp: target_block.header.timestamp.into(), - block_number: target_block_number.into(), + block_number: target_block.header.number.into(), block_difficulty: target_block.header.difficulty.into(), block_random: target_block .header diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 5b4ed5dd9..a4dc7e0c6 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -50,7 +50,8 @@ where { let block = cached_provider .get_block(block_number, BlockTransactionsKind::Full) - .await?; + .await? + .ok_or(anyhow::anyhow!("block not found {}", block_number))?; let (code_db, txn_info) = txn::process_transactions(&block, cached_provider.get_provider().await?.deref()).await?; diff --git a/zero/src/rpc/native/state.rs b/zero/src/rpc/native/state.rs index 3c37e8cbc..b5b82106a 100644 --- a/zero/src/rpc/native/state.rs +++ b/zero/src/rpc/native/state.rs @@ -35,6 +35,7 @@ where let prev_state_root = cached_provider .get_block((block_number - 1).into(), BlockTransactionsKind::Hashes) .await? + .ok_or(anyhow::anyhow!("block not found {}", block_number - 1))? .header .state_root; From 0aad2a7a312a7773df563e746fafbf3b301fd7b6 Mon Sep 17 00:00:00 2001 From: Marko Atanasievski Date: Mon, 28 Oct 2024 13:42:03 +0100 Subject: [PATCH 12/17] chore: update codeowners (#750) * chore: update codeowners * fix: add Paul as second codeowner --- .github/CODEOWNERS | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cc95b3be3..d0b588152 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,9 +1,9 @@ -* @muursh @Nashtare -/evm_arithmetization/ @wborgeaud @muursh @Nashtare @LindaGuiga -/zero/ @muursh @Nashtare @atanmarko -/smt_trie/ @0xaatif @muursh @Nashtare -/mpt_trie/ @0xaatif @Nashtare @muursh -/trace_decoder/ @0xaatif @muursh @Nashtare -.github/ @0xaatif @atanmarko @muursh @Nashtare +* @Nashtare @paulgoleary +/evm_arithmetization/ @wborgeaud @Nashtare @LindaGuiga +/zero/ @Nashtare @atanmarko @0xaatif +/smt_trie/ @0xaatif @Nashtare +/mpt_trie/ @0xaatif @Nashtare +/trace_decoder/ @0xaatif @Nashtare +.github/ @0xaatif @atanmarko @Nashtare /vscode-extension @0xaatif /evm_arithmetization/src/bin/lsp-server.rs @0xaatif From 2b6abb5b50c544fe652585f8d82a99e3ee967224 Mon Sep 17 00:00:00 2001 From: 0xaatif <169152398+0xaatif@users.noreply.github.com> Date: Thu, 31 Oct 2024 00:02:30 +0000 Subject: [PATCH 13/17] ci: warn on outdated top level dependencies (#757) * mark: 0xaatif/outdated * run: cargo init scripts * feat: cargo xtask outdated * ci: lint outdated * chore: update description * fix(ci): pin kurtosis version --- .cargo/config.toml | 3 ++ .github/workflows/jerigon-native.yml | 2 +- .github/workflows/jerigon-zero.yml | 2 +- .github/workflows/lint.yml | 10 ++++ Cargo.lock | 10 ++++ Cargo.toml | 1 + scripts/Cargo.toml | 23 ++++++++++ scripts/xtask.rs | 69 ++++++++++++++++++++++++++++ 8 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 scripts/Cargo.toml create mode 100644 scripts/xtask.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 6340ce34a..ace541bb4 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,3 +2,6 @@ # https://github.com/rust-lang/rust/pull/124129 # https://github.com/dtolnay/linkme/pull/88 rustflags = ["-Z", "linker-features=-lld"] + +[alias] +xtask = ["run", "--package=xtask", "--"] diff --git a/.github/workflows/jerigon-native.yml b/.github/workflows/jerigon-native.yml index 1ef443d9d..dbb7cdecd 100644 --- a/.github/workflows/jerigon-native.yml +++ b/.github/workflows/jerigon-native.yml @@ -52,7 +52,7 @@ jobs: run: | echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list sudo apt update - sudo apt install kurtosis-cli + sudo apt install kurtosis-cli=1.3.1 #It is much easier to use cast tool in scripts so install foundry - name: Install Foundry diff --git a/.github/workflows/jerigon-zero.yml b/.github/workflows/jerigon-zero.yml index e0dc4c40f..034c6b0cd 100644 --- a/.github/workflows/jerigon-zero.yml +++ b/.github/workflows/jerigon-zero.yml @@ -52,7 +52,7 @@ jobs: run: | echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list sudo apt update - sudo apt install kurtosis-cli + sudo apt install kurtosis-cli=1.3.1 #It is much easier to use cast tool in scripts so install foundry - name: Install Foundry diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 13089b3d0..9999d76f8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -66,3 +66,13 @@ jobs: with: tool: taplo-cli - run: taplo fmt --check + outdated: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/rust + - uses: taiki-e/install-action@v2 + with: + tool: cargo-outdated + - run: cargo xtask outdated diff --git a/Cargo.lock b/Cargo.lock index 0dc5ace4d..bc83b02b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5856,6 +5856,16 @@ dependencies = [ "time", ] +[[package]] +name = "xtask" +version = "0.0.0" +dependencies = [ + "anyhow", + "clap", + "serde", + "serde_json", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 3b038ff26..117f124b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "evm_arithmetization", "mpt_trie", "proc_macro", + "scripts", "smt_trie", "trace_decoder", "zero", diff --git a/scripts/Cargo.toml b/scripts/Cargo.toml new file mode 100644 index 000000000..d4328f96c --- /dev/null +++ b/scripts/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "xtask" +version = "0.0.0" +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +clap = { workspace = true, features = ["derive"] } +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true + +[lints] +workspace = true + +[[bin]] +name = "xtask" +path = "xtask.rs" diff --git a/scripts/xtask.rs b/scripts/xtask.rs new file mode 100644 index 000000000..c60770e28 --- /dev/null +++ b/scripts/xtask.rs @@ -0,0 +1,69 @@ +//! General purpose scripts for development + +use std::process::{Command, Stdio}; + +use anyhow::{ensure, Context as _}; +use clap::Parser; +use serde::Deserialize; + +#[derive(Parser)] +enum Args { + /// Run `cargo-outdated`, printing warnings compatible with GitHub's CI. + /// + /// If a direct dependency listed in our Cargo.lock is behind the latest + /// available on crates-io, a warning will be emitted. + /// + /// Note that we only warn on our _direct_ dependencies, + /// not the entire supply chain. + Outdated, +} + +#[derive(Deserialize)] +struct Outdated<'a> { + crate_name: &'a str, + dependencies: Vec>, +} + +#[derive(Deserialize)] +struct Dependency<'a> { + name: &'a str, + project: &'a str, + latest: &'a str, +} + +fn main() -> anyhow::Result<()> { + match Args::parse() { + Args::Outdated => { + let output = Command::new("cargo") + .args(["outdated", "--root-deps-only", "--format=json"]) + .stderr(Stdio::inherit()) + .stdout(Stdio::piped()) + .output() + .context("couldn't exec `cargo`")?; + ensure!( + output.status.success(), + "command failed with {}", + output.status + ); + for Outdated { + crate_name, + dependencies, + } in serde_json::Deserializer::from_slice(&output.stdout) + .into_iter::>() + .collect::, _>>() + .context("failed to parse output from `cargo outdated`")? + { + for Dependency { + name, + project, + latest, + } in dependencies + { + // https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-a-warning-message + println!("::warning title=outdated-dependency::dependency {name} of crate {crate_name} is at version {project}, but the latest is {latest}") + } + } + } + } + Ok(()) +} From 08d447e2c6393b052730f7d2efe877fc56cbdcd7 Mon Sep 17 00:00:00 2001 From: Marko Atanasievski Date: Sun, 3 Nov 2024 23:27:20 +0100 Subject: [PATCH 14/17] chore: remove conditional ci execution (#754) * chore: remove conditional [skip-ci] * fix: test [skip ci] * test: here we want the ci * test: again no execute of tests [skip ci] * test: again, run ci --- .github/workflows/ci.yml | 7 ------- .github/workflows/jerigon-native.yml | 1 - .github/workflows/jerigon-zero.yml | 1 - .github/workflows/pr_checking.yml | 1 + 4 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 030731bdd..6e74116d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,6 @@ jobs: name: Test mpt_trie runs-on: ubuntu-latest timeout-minutes: 30 - if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }} steps: - name: Checkout sources uses: actions/checkout@v4 @@ -50,8 +49,6 @@ jobs: RUST_LOG: info CARGO_INCREMENTAL: 1 RUST_BACKTRACE: 1 - - if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }} steps: - name: Checkout sources uses: actions/checkout@v4 @@ -73,7 +70,6 @@ jobs: name: Test evm_arithmetization runs-on: ubuntu-latest timeout-minutes: 20 - if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }} steps: - name: Checkout sources uses: actions/checkout@v4 @@ -97,7 +93,6 @@ jobs: name: Test zero_bin runs-on: ubuntu-latest timeout-minutes: 10 - if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }} steps: - name: Checkout sources uses: actions/checkout@v4 @@ -112,7 +107,6 @@ jobs: - name: Test in zero_bin subdirectory run: | cargo test --manifest-path zero/Cargo.toml - env: RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0 RUST_LOG: 1 @@ -123,7 +117,6 @@ jobs: name: Test zk_evm_proc_macro runs-on: ubuntu-latest timeout-minutes: 5 - if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }} steps: - name: Checkout sources uses: actions/checkout@v4 diff --git a/.github/workflows/jerigon-native.yml b/.github/workflows/jerigon-native.yml index dbb7cdecd..3105298fa 100644 --- a/.github/workflows/jerigon-native.yml +++ b/.github/workflows/jerigon-native.yml @@ -19,7 +19,6 @@ jobs: name: Native tracer proof generation runs-on: zero-ci timeout-minutes: 30 - if: "! contains(toJSON(github.event.commits.*.message), '[skip-ci]')" steps: - name: Checkout sources uses: actions/checkout@v4 diff --git a/.github/workflows/jerigon-zero.yml b/.github/workflows/jerigon-zero.yml index 034c6b0cd..c58721bfa 100644 --- a/.github/workflows/jerigon-zero.yml +++ b/.github/workflows/jerigon-zero.yml @@ -19,7 +19,6 @@ jobs: name: Zero tracer proof generation runs-on: zero-ci timeout-minutes: 30 - if: "! contains(toJSON(github.event.commits.*.message), '[skip-ci]')" steps: - name: Checkout sources uses: actions/checkout@v4 diff --git a/.github/workflows/pr_checking.yml b/.github/workflows/pr_checking.yml index 497a17208..23c2e2770 100644 --- a/.github/workflows/pr_checking.yml +++ b/.github/workflows/pr_checking.yml @@ -54,3 +54,4 @@ jobs: run: gh pr close ${{ github.event.pull_request.number }} --comment "Spam detected" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + From 5383fbb9f7f8cd619ce17765dfd3911c6e760066 Mon Sep 17 00:00:00 2001 From: Marko Atanasievski Date: Sun, 3 Nov 2024 23:28:03 +0100 Subject: [PATCH 15/17] feat: add yaml linter (#756) * feat: yaml linter * fix: yamllint installation * fix: warnings * fix: multiline command --- .github/actions/rust/action.yml | 6 ++++-- .github/labeler.yml | 22 +++++++++++---------- .github/workflows/audit.yml | 5 ++++- .github/workflows/book.yml | 2 ++ .github/workflows/ci.yml | 10 ++++++++-- .github/workflows/docker_build.yml | 2 ++ .github/workflows/docker_build_push.yml | 2 ++ .github/workflows/jerigon-native.yml | 7 +++++-- .github/workflows/jerigon-zero.yml | 7 +++++-- .github/workflows/labeler.yml | 12 +++++++----- .github/workflows/lint.yml | 2 ++ .github/workflows/pr_checking.yml | 8 ++++++-- .github/workflows/yamllint.yml | 26 +++++++++++++++++++++++++ 13 files changed, 85 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/yamllint.yml diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml index 7df769868..373f93cf0 100644 --- a/.github/actions/rust/action.yml +++ b/.github/actions/rust/action.yml @@ -1,10 +1,12 @@ +--- # Common CI steps + name: "Rust" description: | Common steps for CI - See + runs: - using: composite + using: composite steps: - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: Swatinem/rust-cache@v2 diff --git a/.github/labeler.yml b/.github/labeler.yml index baacbe1fa..04941664b 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,27 +1,29 @@ +--- # Github labeler configuration file + # Add 'crate: mpt_trie' label to any changes within 'mpt_trie' folder. 'crate: mpt_trie': -- changed-files: - - any-glob-to-any-file: mpt_trie/** + - changed-files: + - any-glob-to-any-file: mpt_trie/** # Add 'crate: evm_arithmetization' label to any changes within 'evm_arithmetization' folder. 'crate: evm_arithmetization': -- changed-files: - - any-glob-to-any-file: evm_arithmetization/** + - changed-files: + - any-glob-to-any-file: evm_arithmetization/** # Add 'crate: trace_decoder' label to any changes within 'trace_decoder' folder. 'crate: trace_decoder': -- changed-files: - - any-glob-to-any-file: trace_decoder/** + - changed-files: + - any-glob-to-any-file: trace_decoder/** # Add 'crate: zero_bin' label to any changes within 'zero' folder. 'crate: zero_bin': -- changed-files: - - any-glob-to-any-file: zero/** + - changed-files: + - any-glob-to-any-file: zero/** # Add 'specs' label to any changes within 'docs' or `book` folder. 'specs': -- changed-files: - - any-glob-to-any-file: ['docs/**', 'book/**'] + - changed-files: + - any-glob-to-any-file: ['docs/**', 'book/**'] # Add 'crate: common' label to any changes within 'common' folder. 'crate: common': diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index ee1716de7..dd124c0be 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -1,8 +1,11 @@ +--- # Rust cargo audit workflow + name: Security audit on: push: - paths: + paths: - '**/Cargo.toml' + jobs: security_audit: runs-on: ubuntu-latest diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index c5df23222..b920d424f 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -1,3 +1,5 @@ +--- # MD Book generation and deployment workflow + name: zkEVM mdbook on: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e74116d1..9a45da874 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,3 +1,5 @@ +--- # Workflow with multiple jobs to test different parts of the project + name: Continuous Integration on: @@ -36,7 +38,11 @@ jobs: - name: Test in mpt_trie subdirectory run: cargo test --manifest-path mpt_trie/Cargo.toml env: - RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0 + RUSTFLAGS: > + -Copt-level=3 + -Cdebug-assertions + -Coverflow-checks=y + -Cdebuginfo=0 RUST_LOG: 1 CARGO_INCREMENTAL: 1 RUST_BACKTRACE: 1 @@ -60,7 +66,7 @@ jobs: with: cache-on-failure: true - - name: build # build separately so test logs are actually nice + - name: build # build separately so test logs are actually nice run: cargo build --tests --manifest-path trace_decoder/Cargo.toml - name: test diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml index 1d24cc8b7..e797213aa 100644 --- a/.github/workflows/docker_build.yml +++ b/.github/workflows/docker_build.yml @@ -1,3 +1,5 @@ +--- # Workflow for building and running docker images as regression test + name: Docker Build & Run on: diff --git a/.github/workflows/docker_build_push.yml b/.github/workflows/docker_build_push.yml index 37b89fa92..4aed557df 100644 --- a/.github/workflows/docker_build_push.yml +++ b/.github/workflows/docker_build_push.yml @@ -1,3 +1,5 @@ +--- # Workflow for building and deploying docker images + name: Docker Build & Push on: diff --git a/.github/workflows/jerigon-native.yml b/.github/workflows/jerigon-native.yml index 3105298fa..ccfbdc005 100644 --- a/.github/workflows/jerigon-native.yml +++ b/.github/workflows/jerigon-native.yml @@ -1,3 +1,5 @@ +--- # Run and populate blockchain with transactions and generate proofs using native tracer + name: Jerigon Integration on: @@ -53,14 +55,15 @@ jobs: sudo apt update sudo apt install kurtosis-cli=1.3.1 - #It is much easier to use cast tool in scripts so install foundry + # It is much easier to use cast tool in scripts so install foundry - name: Install Foundry uses: foundry-rs/foundry-toolchain@v1 - name: Run cancun test network run: | docker pull ghcr.io/0xpolygonzero/erigon:feat-zero - kurtosis run --enclave cancun-testnet github.com/ethpandaops/ethereum-package@4.0.0 --args-file jerigon-test-network/network_params.yml + kurtosis run --enclave cancun-testnet github.com/ethpandaops/ethereum-package@4.0.0 \ + --args-file jerigon-test-network/network_params.yml - name: Generate blocks with transactions run: | diff --git a/.github/workflows/jerigon-zero.yml b/.github/workflows/jerigon-zero.yml index c58721bfa..01e580438 100644 --- a/.github/workflows/jerigon-zero.yml +++ b/.github/workflows/jerigon-zero.yml @@ -1,3 +1,5 @@ +--- # Run and populate blockchain with transactions and generate proofs using zero tracer + name: Jerigon Integration on: @@ -53,14 +55,15 @@ jobs: sudo apt update sudo apt install kurtosis-cli=1.3.1 - #It is much easier to use cast tool in scripts so install foundry + # It is much easier to use cast tool in scripts so install foundry - name: Install Foundry uses: foundry-rs/foundry-toolchain@v1 - name: Run cancun test network run: | docker pull ghcr.io/0xpolygonzero/erigon:feat-zero - kurtosis run --enclave cancun-testnet github.com/ethpandaops/ethereum-package@4.0.0 --args-file jerigon-test-network/network_params.yml + kurtosis run --enclave cancun-testnet github.com/ethpandaops/ethereum-package@4.0.0 \ + --args-file jerigon-test-network/network_params.yml - name: Generate blocks with transactions run: | diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index ee79f0f56..9792f107b 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,6 +1,8 @@ +--- # Add labels to the PR + name: "Pull Request Labeler" on: -- pull_request_target + - pull_request_target jobs: labeler: @@ -10,7 +12,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/labeler@v5 - with: - # Allow to remove labels that are no longer relevant when new changes are pushed. - sync-labels: true + - uses: actions/labeler@v5 + with: + # Allow to remove labels that are no longer relevant when new changes are pushed. + sync-labels: true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9999d76f8..e22228b1d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,3 +1,5 @@ +--- # Rust lint related checks + name: lint on: diff --git a/.github/workflows/pr_checking.yml b/.github/workflows/pr_checking.yml index 23c2e2770..10ca3d9fe 100644 --- a/.github/workflows/pr_checking.yml +++ b/.github/workflows/pr_checking.yml @@ -1,3 +1,5 @@ +--- # Rust lint related checks + name: PR check on: @@ -14,7 +16,10 @@ jobs: steps: - name: Set up keywords id: setup_keywords - run: echo "RESTRICTED_KEYWORDS=$(echo '${{ secrets.RESTRICTED_KEYWORDS }}' | jq -r '.[]' | tr '\n' ' ')" >> $GITHUB_ENV + run: > + echo "RESTRICTED_KEYWORDS=$(echo '${{ secrets.RESTRICTED_KEYWORDS }}' + | jq -r '.[]' + | tr '\n' ' ')" >> $GITHUB_ENV - name: Check for spam PR id: check @@ -22,7 +27,6 @@ jobs: # Initialize variables to track spam presence title_is_spam=false description_is_spam=false - # Check title for spam for keyword in $RESTRICTED_KEYWORDS; do if echo "${{ github.event.pull_request.title }}" | grep -i -q "$keyword"; then diff --git a/.github/workflows/yamllint.yml b/.github/workflows/yamllint.yml new file mode 100644 index 000000000..0c9910d31 --- /dev/null +++ b/.github/workflows/yamllint.yml @@ -0,0 +1,26 @@ +--- # Run yamllint on all YAML files in the repository + +name: Yamllint +'on': + pull_request: + paths: + - '**/*.yml' + - '.github/**' + workflow_dispatch: + branches: + - '**' + +jobs: + yamllint-check: + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - name: Install yamllint + run: sudo apt-get install yamllint=1.26.3-1 + - name: Run yamllint + run: > + yamllint + --format github + -d "{extends: default, rules: {line-length: {max: 120}, truthy: {check-keys: false}}}" + .github From a3f9a8dcd5744d141002222f62884b81109cabbf Mon Sep 17 00:00:00 2001 From: Serge Radinovich <47865535+sergerad@users.noreply.github.com> Date: Mon, 4 Nov 2024 15:31:31 +1300 Subject: [PATCH 16/17] Replace regex in trie diff main (#758) * Replace trie diff regex * RM previous regex crate --- Cargo.lock | 25 ++++++++++++++++++++++++- zero/Cargo.toml | 2 +- zero/src/bin/trie_diff.rs | 11 ++++++----- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc83b02b7..d64c3c1b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2861,6 +2861,29 @@ dependencies = [ "waker-fn", ] +[[package]] +name = "lazy-regex" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d8e41c97e6bc7ecb552016274b99fbb5d035e8de288c582d9b933af6677bfda" +dependencies = [ + "lazy-regex-proc_macros", + "once_cell", + "regex", +] + +[[package]] +name = "lazy-regex-proc_macros" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76e1d8b05d672c53cb9c7b920bbba8783845ae4f0b076e02a3db1d02c81b4163" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "syn 2.0.77", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -5893,6 +5916,7 @@ dependencies = [ "itertools 0.13.0", "jemallocator", "keccak-hash 0.10.0", + "lazy-regex", "lru", "mockall", "mpt_trie", @@ -5902,7 +5926,6 @@ dependencies = [ "plonky2", "plonky2_maybe_rayon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "primitive-types 0.12.2", - "regex", "rlp", "ruint", "serde", diff --git a/zero/Cargo.toml b/zero/Cargo.toml index 88c8c580b..96d3bdeba 100644 --- a/zero/Cargo.toml +++ b/zero/Cargo.toml @@ -26,6 +26,7 @@ hashbrown.workspace = true hex.workspace = true itertools.workspace = true keccak-hash.workspace = true +lazy-regex = "3.3.0" lru.workspace = true mpt_trie.workspace = true num-traits.workspace = true @@ -33,7 +34,6 @@ once_cell.workspace = true paladin-core.workspace = true plonky2.workspace = true plonky2_maybe_rayon.workspace = true -regex = "1.5.4" rlp.workspace = true ruint = { workspace = true, features = ["num-traits", "primitive-types"] } serde.workspace = true diff --git a/zero/src/bin/trie_diff.rs b/zero/src/bin/trie_diff.rs index e580684d2..480441486 100644 --- a/zero/src/bin/trie_diff.rs +++ b/zero/src/bin/trie_diff.rs @@ -20,9 +20,9 @@ use anyhow::Result; use clap::{Parser, ValueHint}; use evm_arithmetization::generation::DebugOutputTries; use futures::{future, TryStreamExt}; +use lazy_regex::regex_captures; use paladin::directive::{Directive, IndexedStream}; use paladin::runtime::Runtime; -use regex::Regex; use trace_decoder::observer::TriesObserver; use tracing::{error, info}; use zero::ops::register; @@ -132,10 +132,11 @@ async fn main() -> Result<()> { { // Try to parse block and batch index from error message. let error_message = e2.to_string(); - let re = Regex::new(r"block:(\d+) batch:(\d+)")?; - if let Some(cap) = re.captures(&error_message) { - let block_number: u64 = cap[1].parse()?; - let batch_index: usize = cap[2].parse()?; + if let Some((_, block_number, block_index)) = + regex_captures!(r"block:(\d+) batch:(\d+)", error_message.as_str()) + { + let block_number: u64 = block_number.parse()?; + let batch_index: usize = block_index.parse()?; let prover_tries = zero::debug_utils::load_tries_from_disk(block_number, batch_index)?; From c38c0c67ac7d9c9cc2b570ff3cc3240f29a1505c Mon Sep 17 00:00:00 2001 From: Marko Atanasievski Date: Mon, 4 Nov 2024 10:36:53 +0100 Subject: [PATCH 17/17] feat: add ci shellcheck (#753) * feat: add ci shellcheck * fix: run on scripts change * test: if ci shellcheck would run * chore: pin the shellcheck version * test: shellckeck * fix: add manual workflow execution * fix: runner * pin: 22.04 * chore: run always in PR if some script is changed * fix: review * fix: shellcheck --- .github/workflows/pr_checking.yml | 1 - .github/workflows/shellcheck.yml | 26 ++++++++++++++++ scripts/prove_stdio.sh | 49 +++++++++++++++++-------------- 3 files changed, 53 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/shellcheck.yml diff --git a/.github/workflows/pr_checking.yml b/.github/workflows/pr_checking.yml index 10ca3d9fe..adc6d81bc 100644 --- a/.github/workflows/pr_checking.yml +++ b/.github/workflows/pr_checking.yml @@ -58,4 +58,3 @@ jobs: run: gh pr close ${{ github.event.pull_request.number }} --comment "Spam detected" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml new file mode 100644 index 000000000..e7b826555 --- /dev/null +++ b/.github/workflows/shellcheck.yml @@ -0,0 +1,26 @@ +--- # Workflow to run shellckeck on all shell scripts + +name: Shellcheck +on: + pull_request: + paths: + - '**/*.sh' + - 'scripts/**' + workflow_dispatch: + branches: + - "**" + +jobs: + shellcheck_scripts: + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: Install shellcheck + run: sudo apt-get install shellcheck=0.8.0-2 + + - name: Run shellcheck + run: | + cd scripts + shellcheck ./*.sh diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index c3c794086..b0d0788ca 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -95,15 +95,22 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --quiet --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE &> $TEST_OUT_PATH - if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then + cargo run --quiet --release --package zero --bin leader -- \ + --test-only \ + --runtime in-memory \ + --load-strategy on-demand \ + --block-batch-size "$BLOCK_BATCH_SIZE" \ + --proof-output-dir "$PROOF_OUTPUT_DIR" \ + stdio < "$INPUT_FILE" &> "$TEST_OUT_PATH" + + if grep -q 'All proof witnesses have been generated successfully.' "$TEST_OUT_PATH"; then echo -e "\n\nSuccess - Note this was just a test, not a proof" - rm $TEST_OUT_PATH + rm "$TEST_OUT_PATH" exit else # Some error occurred, display the logs and exit. - cat $OUT_LOG_PATH - echo "Failed to create proof witnesses. See $OUT_LOG_PATH for more details." + cat "$TEST_OUT_PATH" + echo "Failed to create proof witnesses. See $TEST_OUT_PATH for more details." exit 1 fi fi @@ -112,45 +119,43 @@ cargo build --release --jobs "$num_procs" start_time=$(date +%s%N) -"${REPO_ROOT}/target/release/leader" --runtime in-memory --load-strategy on-demand -n 1 --block-batch-size $BLOCK_BATCH_SIZE \ - --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE &> $OUTPUT_LOG +"${REPO_ROOT}/target/release/leader" --runtime in-memory \ + --load-strategy on-demand -n 1 \ + --block-batch-size "$BLOCK_BATCH_SIZE" \ + --proof-output-dir "$PROOF_OUTPUT_DIR" stdio < "$INPUT_FILE" &> "$OUTPUT_LOG" end_time=$(date +%s%N) -cat $OUTPUT_LOG | grep "Successfully wrote to disk proof file " | awk '{print $NF}' | tee $PROOFS_FILE_LIST +grep "Successfully wrote to disk proof file " "$OUTPUT_LOG" | awk '{print $NF}' | tee "$PROOFS_FILE_LIST" if [ ! -s "$PROOFS_FILE_LIST" ]; then # Some error occurred, display the logs and exit. - cat $OUTPUT_LOG + cat "$OUTPUT_LOG" echo "Proof list not generated, some error happened. For more details check the log file $OUTPUT_LOG" exit 1 fi -cat $PROOFS_FILE_LIST | while read proof_file; +while read -r proof_file; do echo "Verifying proof file $proof_file" - verify_file=$PROOF_OUTPUT_DIR/verify_$(basename $proof_file).out - "${REPO_ROOT}/target/release/verifier" -f $proof_file | tee $verify_file - if grep -q 'All proofs verified successfully!' $verify_file; then + verify_file=$PROOF_OUTPUT_DIR/verify_$(basename "$proof_file").out + "${REPO_ROOT}/target/release/verifier" -f "$proof_file" | tee "$verify_file" + if grep -q 'All proofs verified successfully!' "$verify_file"; then echo "Proof verification for file $proof_file successful"; - rm $verify_file # we keep the generated proof for potential reuse + rm "$verify_file" # we keep the generated proof for potential reuse else # Some error occurred with verification, display the logs and exit. - cat $verify_file + cat "$verify_file" echo "There was an issue with proof verification. See $verify_file for more details."; exit 1 fi -done +done < "$PROOFS_FILE_LIST" duration_ns=$((end_time - start_time)) duration_sec=$(echo "$duration_ns / 1000000000" | bc -l) echo "Success!" -echo "Proving duration:" $duration_sec " seconds" +echo "Proving duration: $duration_sec seconds" echo "Note, this duration is inclusive of circuit handling and overall process initialization"; # Clean up in case of success -rm $OUTPUT_LOG - - - - +rm "$OUTPUT_LOG"