diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..6340ce34a --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,4 @@ +[build] +# https://github.com/rust-lang/rust/pull/124129 +# https://github.com/dtolnay/linkme/pull/88 +rustflags = ["-Z", "linker-features=-lld"] diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..1562ab5b3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,28 @@ +# this is loosely based on `docker init`'s rust template. + +**/.DS_Store +**/.classpath +**/.dockerignore +# **/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/charts +**/docker-compose* +**/compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/secrets.dev.yaml +**/values.dev.yaml +/bin +/target +LICENSE +README.md diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml index 8d74a2fa4..c907bbf39 100644 --- a/.github/workflows/docker_build.yml +++ b/.github/workflows/docker_build.yml @@ -12,24 +12,15 @@ on: jobs: docker: - name: Build and run leader and worker docker images for regression check + name: Regression test docker images runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - - name: Build leader docker container - run: | - docker build --progress plain -t leader:${{ github.ref_name }} -f leader.Dockerfile . - - - name: Run leader docker container - run: | - docker run --rm leader:${{ github.ref_name }} --help - - - name: Build worker docker container - run: | - docker build --progress plain -t worker:${{ github.ref_name }} -f worker.Dockerfile . - - - name: Run worker docker container - run: | - docker run --rm worker:${{ github.ref_name }} --help + - run: | + docker build --progress=plain --build-arg=PROFILE=dev --tag scratch . + docker run --rm --init --entrypoint leader scratch --help + docker run --rm --init --entrypoint worker scratch --help + docker run --rm --init --entrypoint rpc scratch --help + docker run --rm --init --entrypoint verifier scratch --help diff --git a/.github/workflows/docker_build_push.yml b/.github/workflows/docker_build_push.yml index 112ea158e..0c64efecd 100644 --- a/.github/workflows/docker_build_push.yml +++ b/.github/workflows/docker_build_push.yml @@ -8,8 +8,6 @@ on: env: REGISTRY: ghcr.io - IMAGE_NAME_LEADER: ${{ github.repository }}-leader - IMAGE_NAME_WORKER: ${{ github.repository }}-worker jobs: docker: @@ -34,50 +32,24 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Leader Docker - id: meta_leader - uses: docker/metadata-action@v5 - with: - images: | - name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LEADER }} - tags: | - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - - - name: Push to GitHub Container Registry - Leader - uses: docker/build-push-action@v3 - with: - context: . - file: ./leader.Dockerfile - push: true - # platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta_leader.outputs.tags }} - labels: ${{ steps.meta_leader.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Extract metadata (tags, labels) for Worker Docker - id: meta_worker + - name: Extract metadata (tags, labels) + id: meta uses: docker/metadata-action@v5 with: images: | - name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_WORKER }} + name=${{ env.REGISTRY }}/${{ github.repository }} tags: | type=ref,event=branch type=ref,event=pr type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} - - name: Push to GitHub Container Registry - Worker + - name: Push to GitHub Container Registry uses: docker/build-push-action@v3 with: - context: . - file: ./worker.Dockerfile push: true # platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta_worker.outputs.tags }} - labels: ${{ steps.meta_worker.outputs.labels }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/Cargo.lock b/Cargo.lock index c62bf3dca..6b78f1e7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1161,6 +1161,38 @@ dependencies = [ "serde", ] +[[package]] +name = "camino" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.23", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cast" version = "0.3.0" @@ -2814,6 +2846,7 @@ dependencies = [ "alloy", "anyhow", "axum", + "cargo_metadata", "clap", "dotenvy", "futures", @@ -4373,6 +4406,9 @@ name = "semver" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +dependencies = [ + "serde", +] [[package]] name = "semver-parser" diff --git a/Cargo.toml b/Cargo.toml index b64e627fb..1ff61b81d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,18 +1,20 @@ [workspace] -members = ["mpt_trie", - "smt_trie", - "proof_gen", - "trace_decoder", +members = [ + "compat", "evm_arithmetization", + "mpt_trie", "proc_macro", - "zero_bin/leader", - "zero_bin/worker", + "proof_gen", + "smt_trie", + "trace_decoder", "zero_bin/common", + "zero_bin/leader", "zero_bin/ops", - "zero_bin/verifier", + "zero_bin/prover", "zero_bin/rpc", - "zero_bin/prover", - "compat"] + "zero_bin/verifier", + "zero_bin/worker", +] resolver = "2" [workspace.package] @@ -24,7 +26,7 @@ keywords = ["cryptography", "STARK", "plonky2", "ethereum", "zk"] categories = ["cryptography::cryptocurrencies"] [workspace.dependencies] -alloy = { git = "https://github.com/alloy-rs/alloy", tag='v0.1.1', default-features = false, features = [ +alloy = { git = "https://github.com/alloy-rs/alloy", tag = 'v0.1.1', default-features = false, features = [ "consensus", "reqwest", "json-rpc", @@ -36,7 +38,7 @@ alloy = { git = "https://github.com/alloy-rs/alloy", tag='v0.1.1', default-featu "providers", "transports", "transport-http", - "rpc-types-debug" + "rpc-types-debug", ] } anyhow = "1.0.86" async-stream = "0.3.5" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..18158356f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,100 @@ +# syntax=docker/dockerfile:1 +# This is loosely based on `docker init`'s rust template. +# For a completely clean build, run something like this: +# ``` +# docker build --build-arg=PROFILE=dev --no-cache +# ``` + +############# +# Build stage +############# +# - `/src` is the repo directory. +# - `/artifacts` is $CARGO_TARGET_DIR. +# - `/output` is where the binaries go. + +ARG BUILD_BASE=rustlang/rust:nightly-bullseye-slim +FROM ${BUILD_BASE} AS build + +# Install build dependencies. +RUN apt-get update && apt-get install -y \ + # for jemalloc + libjemalloc-dev \ + libjemalloc2 \ + make \ + # for openssl + libssl-dev \ + pkg-config \ + # clean the image + && rm -rf /var/lib/apt/lists/* + +ARG PROFILE=release +# forward the docker argument so that the script below can read it +ENV PROFILE=${PROFILE} + +# Build the application. +RUN \ + # mount the repository so we don't have to COPY it in + --mount=type=bind,source=.,target=/src \ + # cache artifacts and the cargo registry to speed up subsequent builds + --mount=type=cache,target=/artifacts \ + --mount=type=cache,target=/usr/local/cargo/registry/ \ + # run the build + <, const D: usize> BytePackingStark { ops: Vec, min_rows: usize, ) -> Vec<[F; NUM_COLUMNS]> { - let base_len: usize = ops.iter().map(|op| usize::from(!op.bytes.is_empty())).sum(); - let num_rows = core::cmp::max(base_len.max(BYTE_RANGE_MAX), min_rows).next_power_of_two(); + let num_rows = core::cmp::max(ops.len().max(BYTE_RANGE_MAX), min_rows).next_power_of_two(); let mut rows = Vec::with_capacity(num_rows); for op in ops { diff --git a/evm_arithmetization/src/cpu/columns/general.rs b/evm_arithmetization/src/cpu/columns/general.rs index 9e2713b9e..125e7e18f 100644 --- a/evm_arithmetization/src/cpu/columns/general.rs +++ b/evm_arithmetization/src/cpu/columns/general.rs @@ -2,8 +2,11 @@ use core::borrow::{Borrow, BorrowMut}; use core::fmt::{Debug, Formatter}; use core::mem::{size_of, transmute}; +use static_assertions::const_assert; + /// General purpose columns, which can have different meanings depending on what /// CTL or other operation is occurring at this row. +#[repr(C)] #[derive(Clone, Copy)] pub(crate) union CpuGeneralColumnsView { exception: CpuExceptionView, @@ -11,6 +14,7 @@ pub(crate) union CpuGeneralColumnsView { jumps: CpuJumpsView, shift: CpuShiftView, stack: CpuStackView, + push: CpuPushView, } impl CpuGeneralColumnsView { @@ -75,6 +79,18 @@ impl CpuGeneralColumnsView { pub(crate) fn stack_mut(&mut self) -> &mut CpuStackView { unsafe { &mut self.stack } } + + /// View of the columns required for the push operation. + /// SAFETY: Each view is a valid interpretation of the underlying array. + pub(crate) fn push(&self) -> &CpuPushView { + unsafe { &self.push } + } + + /// Mutable view of the columns required for the push operation. + /// SAFETY: Each view is a valid interpretation of the underlying array. + pub(crate) fn push_mut(&mut self) -> &mut CpuPushView { + unsafe { &mut self.push } + } } impl PartialEq for CpuGeneralColumnsView { @@ -107,14 +123,21 @@ impl BorrowMut<[T; NUM_SHARED_COLUMNS]> for CpuGeneralColumnsView { } /// View of the first three `CpuGeneralColumns` containing exception code bits. +#[repr(C)] #[derive(Copy, Clone)] pub(crate) struct CpuExceptionView { /// Exception code as little-endian bits. pub(crate) exc_code_bits: [T; 3], + /// Reserve the unused columns. + _padding_columns: [T; NUM_SHARED_COLUMNS - 3], } /// View of the `CpuGeneralColumns` storing pseudo-inverses used to prove logic /// operations. +/// +/// Because this is the largest field of the [`CpuGeneralColumnsView`] union, +/// we don't add any padding columns. +#[repr(C)] #[derive(Copy, Clone)] pub(crate) struct CpuLogicView { /// Pseudoinverse of `(input0 - input1)`. Used prove that they are unequal. @@ -124,30 +147,40 @@ pub(crate) struct CpuLogicView { /// View of the first two `CpuGeneralColumns` storing a flag and a pseudoinverse /// used to prove jumps. +#[repr(C)] #[derive(Copy, Clone)] pub(crate) struct CpuJumpsView { /// A flag indicating whether a jump should occur. pub(crate) should_jump: T, /// Pseudoinverse of `cond.iter().sum()`. Used to check `should_jump`. pub(crate) cond_sum_pinv: T, + /// Reserve the unused columns. + _padding_columns: [T; NUM_SHARED_COLUMNS - 2], } /// View of the first `CpuGeneralColumns` storing a pseudoinverse used to prove /// shift operations. +#[repr(C)] #[derive(Copy, Clone)] pub(crate) struct CpuShiftView { /// For a shift amount of displacement: [T], this is the inverse of /// sum(displacement[1..]) or zero if the sum is zero. pub(crate) high_limb_sum_inv: T, + /// Reserve the unused columns. + _padding_columns: [T; NUM_SHARED_COLUMNS - 1], } /// View of the last four `CpuGeneralColumns` storing stack-related variables. /// The first three are used for conditionally enabling and disabling channels /// when reading the next `stack_top`, and the fourth one is used to check for /// stack overflow. +#[repr(C)] #[derive(Copy, Clone)] pub(crate) struct CpuStackView { - _unused: [T; 4], + /// Reserve the unused columns at the beginning. This allows `Self` to + /// coexist with any view that uses only the first four columns (i.e. all + /// except `CpuLogicView`). + _unused: [T; NUM_SHARED_COLUMNS - 4], /// Pseudoinverse of `stack_len - num_pops`. pub(crate) stack_inv: T, /// stack_inv * stack_len. @@ -159,6 +192,30 @@ pub(crate) struct CpuStackView { pub(crate) stack_len_bounds_aux: T, } -/// Number of columns shared by all the views of `CpuGeneralColumnsView`. -/// `u8` is guaranteed to have a `size_of` of 1. -pub(crate) const NUM_SHARED_COLUMNS: usize = size_of::>(); +/// View of the first `CpuGeneralColumn` storing the negation of +/// `is_kernel_mode` flag, to filter out `PUSH` instructions from being +/// range-checked when happening in the KERNEL context. +#[repr(C)] +#[derive(Copy, Clone)] +pub(crate) struct CpuPushView { + /// Product of `push_prover_input` with the negated `is_kernel_mode` flag. + pub(crate) is_not_kernel: T, + /// Reserve the unused columns. + _padding_columns: [T; NUM_SHARED_COLUMNS - 1], +} +/// The number of columns shared by all views of [`CpuGeneralColumnsView`]. +/// This is defined in terms of the largest view in order to determine the +/// number of padding columns to add to each field without creating a cycle +/// for rustc. +/// NB: `u8` is guaranteed to have a `size_of` of 1. +pub(crate) const NUM_SHARED_COLUMNS: usize = size_of::>(); +const_assert!(NUM_SHARED_COLUMNS == size_of::>()); + +// Assert that each field of the [`CpuGeneralColumnsView`] union contains the +// correct number of columns. +const_assert!(size_of::>() == NUM_SHARED_COLUMNS); +const_assert!(size_of::>() == NUM_SHARED_COLUMNS); +const_assert!(size_of::>() == NUM_SHARED_COLUMNS); +const_assert!(size_of::>() == NUM_SHARED_COLUMNS); +const_assert!(size_of::>() == NUM_SHARED_COLUMNS); +const_assert!(size_of::>() == NUM_SHARED_COLUMNS); diff --git a/evm_arithmetization/src/cpu/control_flow.rs b/evm_arithmetization/src/cpu/control_flow.rs index c4fcb91ec..ba4e71890 100644 --- a/evm_arithmetization/src/cpu/control_flow.rs +++ b/evm_arithmetization/src/cpu/control_flow.rs @@ -78,6 +78,13 @@ pub(crate) fn eval_packed_generic( ); yield_constr.constraint_transition(is_prover_input * (lv.is_kernel_mode - nv.is_kernel_mode)); + // Check the helper value in the general columns. We do not need to enforce that + // it is set to 0 if the operation is a `PROVER_INPUT`, as the latter is a + // kernel-only instruction. This is enforced in the `decode` module. + yield_constr.constraint( + lv.op.push_prover_input * ((lv.is_kernel_mode + lv.general.push().is_not_kernel) - P::ONES), + ); + // If a non-CPU cycle row is followed by a CPU cycle row, then: // - the `program_counter` of the CPU cycle row is `main` (the entry point of // our kernel), @@ -136,6 +143,16 @@ pub(crate) fn eval_ext_circuit, const D: usize>( yield_constr.constraint_transition(builder, kernel_constr); } + // Check the helper value in the general columns. We do not need to enforce that + // it is set to 0 if the operation is a `PROVER_INPUT`, as the latter is a + // kernel-only instruction. This is enforced in the `decode` module. + { + let temp = builder.add_extension(lv.is_kernel_mode, lv.general.push().is_not_kernel); + let constr = + builder.mul_sub_extension(lv.op.push_prover_input, temp, lv.op.push_prover_input); + yield_constr.constraint(builder, constr); + } + // If a non-CPU cycle row is followed by a CPU cycle row, then: // - the `program_counter` of the CPU cycle row is `main` (the entry point of // our kernel), diff --git a/evm_arithmetization/src/cpu/cpu_stark.rs b/evm_arithmetization/src/cpu/cpu_stark.rs index 958dd1267..21c61f1e6 100644 --- a/evm_arithmetization/src/cpu/cpu_stark.rs +++ b/evm_arithmetization/src/cpu/cpu_stark.rs @@ -260,13 +260,11 @@ pub(crate) fn ctl_data_byte_packing_push() -> Vec> { /// CTL filter for the `PUSH` operation. pub(crate) fn ctl_filter_byte_packing_push() -> Filter { - let col_bit = Column::linear_combination_with_constant( - vec![(COL_MAP.opcode_bits[7], F::NEG_ONE)], - F::ONE, - ); - Filter::new( - vec![(Column::single(COL_MAP.op.push_prover_input), col_bit)], + vec![( + Column::single(COL_MAP.general.push().is_not_kernel), + Column::single(COL_MAP.op.push_prover_input), + )], vec![], ) } diff --git a/evm_arithmetization/src/witness/operation.rs b/evm_arithmetization/src/witness/operation.rs index 46b1b4b10..ac884c74a 100644 --- a/evm_arithmetization/src/witness/operation.rs +++ b/evm_arithmetization/src/witness/operation.rs @@ -3,6 +3,7 @@ use itertools::Itertools; use keccak_hash::keccak; use plonky2::field::types::Field; +use super::state::KERNEL_CONTEXT; use super::transition::Transition; use super::util::{ byte_packing_log, byte_unpacking_log, mem_read_with_log, mem_write_log, @@ -386,7 +387,13 @@ pub(crate) fn generate_push>( let val = U256::from_big_endian(&bytes); push_with_write(state, &mut row, val)?; - byte_packing_log(state, base_address, bytes); + // This is necessary to filter out PUSH instructions from the BytePackingStark + // CTl when happening in the KERNEL context. + row.general.push_mut().is_not_kernel = F::ONE - row.is_kernel_mode; + + if code_context != KERNEL_CONTEXT { + byte_packing_log(state, base_address, bytes); + } state.push_cpu(row); diff --git a/evm_arithmetization/src/witness/state.rs b/evm_arithmetization/src/witness/state.rs index 1070ee643..5edcf69bb 100644 --- a/evm_arithmetization/src/witness/state.rs +++ b/evm_arithmetization/src/witness/state.rs @@ -2,7 +2,7 @@ use ethereum_types::U256; use crate::cpu::kernel::aggregator::KERNEL; -const KERNEL_CONTEXT: usize = 0; +pub(crate) const KERNEL_CONTEXT: usize = 0; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct RegistersState { diff --git a/evm_arithmetization/src/witness/traces.rs b/evm_arithmetization/src/witness/traces.rs index 271595712..a8cdfa454 100644 --- a/evm_arithmetization/src/witness/traces.rs +++ b/evm_arithmetization/src/witness/traces.rs @@ -65,11 +65,7 @@ impl Traces { Operation::RangeCheckOperation { .. } => 1, }) .sum(), - byte_packing_len: self - .byte_packing_ops - .iter() - .map(|op| usize::from(!op.bytes.is_empty())) - .sum(), + byte_packing_len: self.byte_packing_ops.len(), cpu_len: self.cpu.len(), keccak_len: self.keccak_inputs.len() * keccak::keccak_stark::NUM_ROUNDS, keccak_sponge_len: self diff --git a/evm_arithmetization/src/witness/util.rs b/evm_arithmetization/src/witness/util.rs index b90541e69..090a13981 100644 --- a/evm_arithmetization/src/witness/util.rs +++ b/evm_arithmetization/src/witness/util.rs @@ -344,6 +344,11 @@ pub(crate) fn byte_packing_log>( base_address: MemoryAddress, bytes: Vec, ) { + if bytes.is_empty() { + // No-op + return; + } + let clock = state.get_clock(); let mut address = base_address; diff --git a/leader.Dockerfile b/leader.Dockerfile deleted file mode 100644 index ef69a430f..000000000 --- a/leader.Dockerfile +++ /dev/null @@ -1,60 +0,0 @@ -FROM rustlang/rust:nightly-bullseye-slim as builder - -RUN apt-get update && apt-get install -y libjemalloc2 libjemalloc-dev make libssl-dev pkg-config - -RUN mkdir -p zero_bin -COPY Cargo.toml . -# Cleanup all workspace members and add selected crates again -RUN sed -i '/members =/{:a;N;/]/!ba};//d' Cargo.toml -RUN sed -i 's#\[workspace\]#\[workspace\]\nmembers = \["zero_bin\/leader", "zero_bin\/prover", "zero_bin\/rpc", "zero_bin\/common", \ - "zero_bin\/ops"\, "evm_arithmetization", "trace_decoder", "mpt_trie", "proc_macro", "compat"\]#' Cargo.toml -COPY Cargo.lock . -COPY ./rust-toolchain.toml ./ -RUN cat ./Cargo.toml -COPY ./.env ./.env - -COPY proof_gen proof_gen -COPY mpt_trie mpt_trie -COPY proc_macro proc_macro -COPY compat compat -COPY trace_decoder trace_decoder -COPY evm_arithmetization evm_arithmetization -COPY zero_bin/common zero_bin/common -COPY zero_bin/ops zero_bin/ops -COPY zero_bin/rpc zero_bin/rpc -COPY zero_bin/prover zero_bin/prover -COPY zero_bin/leader zero_bin/leader - - -RUN \ - touch zero_bin/common/src/lib.rs && \ - touch zero_bin/ops/src/lib.rs && \ - touch zero_bin/leader/src/main.rs && \ - touch zero_bin/rpc/src/lib.rs && \ - touch zero_bin/prover/src/lib.rs && \ - touch evm_arithmetization/src/lib.rs && \ - touch trace_decoder/src/lib.rs && \ - touch mpt_trie/src/lib.rs && \ - touch proc_macro/src/lib.rs && \ - touch compat/src/lib.rs - -# Disable the lld linker for now, as it's causing issues with the linkme package. -# https://github.com/rust-lang/rust/pull/124129 -# https://github.com/dtolnay/linkme/pull/88 -ENV RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' - -RUN cargo build --release --bin leader -RUN cargo build --release --bin rpc - - -FROM debian:bullseye-slim -RUN apt-get update && apt-get install -y ca-certificates libjemalloc2 -COPY --from=builder ./target/release/leader /usr/local/bin/leader -COPY --from=builder ./target/release/rpc /usr/local/bin/rpc -COPY --from=builder ./.env /.env - -# Workaround for the issue with the Cargo.lock search path -# Related to issue https://github.com/0xPolygonZero/zk_evm/issues/311 -RUN mkdir -p zero_bin/leader - -ENTRYPOINT ["/usr/local/bin/leader"] diff --git a/trace_decoder/src/decoding.rs b/trace_decoder/src/decoding.rs index 5acfca45f..eec2efbac 100644 --- a/trace_decoder/src/decoding.rs +++ b/trace_decoder/src/decoding.rs @@ -31,7 +31,7 @@ use crate::{ OtherBlockData, TrieRootHash, TxnIdx, EMPTY_ACCOUNT_BYTES_RLPED, ZERO_STORAGE_SLOT_VAL_RLPED, }, - utils::{hash, optional_field, optional_field_hex, update_val_if_some}, + utils::{eth_to_gwei, hash, optional_field, optional_field_hex, update_val_if_some}, }; /// Stores the result of parsing tries. Returns a [TraceParsingError] upon @@ -631,8 +631,13 @@ impl ProcessedBlockTrace { fn add_withdrawals_to_txns( txn_ir: &mut [GenerationInputs], final_trie_state: &mut PartialTrieState, - withdrawals: Vec<(Address, U256)>, + mut withdrawals: Vec<(Address, U256)>, ) -> TraceParsingResult<()> { + // Scale withdrawals amounts. + for (_addr, amt) in withdrawals.iter_mut() { + *amt = eth_to_gwei(*amt) + } + let withdrawals_with_hashed_addrs_iter = || { withdrawals .iter() diff --git a/trace_decoder/src/utils.rs b/trace_decoder/src/utils.rs index 7abfe00df..2995ca68d 100644 --- a/trace_decoder/src/utils.rs +++ b/trace_decoder/src/utils.rs @@ -1,4 +1,4 @@ -use ethereum_types::H256; +use ethereum_types::{H256, U256}; use keccak_hash::keccak; use log::trace; use mpt_trie::{ @@ -8,6 +8,11 @@ use mpt_trie::{ use crate::types::HashedStorageAddr; +pub(crate) fn eth_to_gwei(eth: U256) -> U256 { + // 1 ether = 10^9 gwei. + eth * U256::from(10).pow(9.into()) +} + pub(crate) fn hash(bytes: &[u8]) -> H256 { H256::from(keccak(bytes).0) } diff --git a/worker.Dockerfile b/worker.Dockerfile deleted file mode 100644 index 865fb7bb6..000000000 --- a/worker.Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -FROM rustlang/rust:nightly-bullseye-slim as builder - -RUN apt-get update && apt-get install -y libjemalloc2 libjemalloc-dev make libssl-dev pkg-config - -RUN mkdir -p zero_bin -COPY Cargo.toml . -# Cleanup all workspace members and add selected crates again -RUN sed -i '/members =/{:a;N;/]/!ba};//d' Cargo.toml -RUN sed -i 's#\[workspace\]#\[workspace\]\nmembers = \["zero_bin\/worker", "zero_bin\/common", "zero_bin\/ops"\, "evm_arithmetization", "mpt_trie", "proc_macro"\]#' Cargo.toml -COPY Cargo.lock . -COPY ./rust-toolchain.toml ./ - -COPY proof_gen proof_gen -COPY mpt_trie mpt_trie -COPY evm_arithmetization evm_arithmetization -COPY proc_macro proc_macro -COPY zero_bin/common zero_bin/common -COPY zero_bin/ops zero_bin/ops -COPY zero_bin/worker zero_bin/worker - -RUN \ - touch zero_bin/common/src/lib.rs && \ - touch zero_bin/ops/src/lib.rs && \ - touch zero_bin/worker/src/main.rs && \ - touch evm_arithmetization/src/lib.rs && \ - touch mpt_trie/src/lib.rs && \ - touch proc_macro/src/lib.rs - -# Disable the lld linker for now, as it's causing issues with the linkme package. -# https://github.com/rust-lang/rust/pull/124129 -# https://github.com/dtolnay/linkme/pull/88 -ENV RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' - -RUN cargo build --release --bin worker - -FROM debian:bullseye-slim -RUN apt-get update && apt-get install -y ca-certificates libjemalloc2 -COPY --from=builder ./target/release/worker /usr/local/bin/worker -ENTRYPOINT ["/usr/local/bin/worker"] - diff --git a/zero_bin/.cargo/config.toml b/zero_bin/.cargo/config.toml index b9c0d3375..f4ac0fbdd 100644 --- a/zero_bin/.cargo/config.toml +++ b/zero_bin/.cargo/config.toml @@ -1,8 +1,4 @@ -[build] -# https://github.com/rust-lang/rust/pull/124129 -# https://github.com/dtolnay/linkme/pull/88 -rustflags = ["-Zlinker-features=-lld"] - [env] -# If we're running in the project workspace, then we should set the workspace env var so we read/write to/from files relative to the workspace. +# read/write to/from files relative to the workspace even when running in this +# directory CARGO_WORKSPACE_DIR = { value = "", relative = true } diff --git a/zero_bin/leader/Cargo.toml b/zero_bin/leader/Cargo.toml index 10a3e20f5..dfb0070b4 100644 --- a/zero_bin/leader/Cargo.toml +++ b/zero_bin/leader/Cargo.toml @@ -29,8 +29,12 @@ toml = { workspace = true } ops = { workspace = true } prover = { workspace = true } rpc = { workspace = true } -zero_bin_common ={ workspace = true } +zero_bin_common = { workspace = true } [features] default = [] test_only = ["ops/test_only", "prover/test_only"] + +[build-dependencies] +cargo_metadata = "0.18.1" +anyhow = { workspace = true } diff --git a/zero_bin/leader/build.rs b/zero_bin/leader/build.rs new file mode 100644 index 000000000..0adcbb347 --- /dev/null +++ b/zero_bin/leader/build.rs @@ -0,0 +1,19 @@ +use anyhow::Context as _; +fn main() -> anyhow::Result<()> { + let meta = cargo_metadata::MetadataCommand::new() + .exec() + .context("failed to probe cargo-metadata")?; + let version = &meta + .packages + .iter() + .find(|it| it.name == "evm_arithmetization") + .context("couldn't find evm_arithmetization package")? + .version; + println!( + "cargo::rustc-env=EVM_ARITHMETIZATION_PACKAGE_VERSION={}.{}.x", + // patch version change should not prompt circuits regeneration + version.major, + version.minor + ); + Ok(()) +} diff --git a/zero_bin/leader/src/main.rs b/zero_bin/leader/src/main.rs index a2c7020f1..94b2abb94 100644 --- a/zero_bin/leader/src/main.rs +++ b/zero_bin/leader/src/main.rs @@ -13,14 +13,14 @@ use tracing::{info, warn}; use zero_bin_common::block_interval::BlockInterval; use crate::client::{client_main, ProofParams}; -use crate::utils::get_package_version; mod cli; mod client; mod http; mod init; mod stdio; -mod utils; + +const EVM_ARITH_VER_KEY: &str = "EVM_ARITHMETIZATION_PKG_VER"; fn get_previous_proof(path: Option) -> Result> { if path.is_none() { @@ -39,24 +39,17 @@ async fn main() -> Result<()> { load_dotenvy_vars_if_present(); init::tracing(); - if env::var("EVM_ARITHMETIZATION_PKG_VER").is_err() { - let pkg_ver = get_package_version("evm_arithmetization")?; - // Extract the major and minor version parts and append 'x' as the patch version - if let Some((major_minor, _)) = pkg_ver.as_ref().and_then(|s| s.rsplit_once('.')) { - let circuits_version = format!("{}.x", major_minor); - // Set the environment variable for the evm_arithmetization package version - #[allow(unused_unsafe)] - unsafe { - env::set_var("EVM_ARITHMETIZATION_PKG_VER", circuits_version); - } - } else { - // Set to "NA" if version extraction fails - #[allow(unused_unsafe)] - unsafe { - env::set_var("EVM_ARITHMETIZATION_PKG_VER", "NA"); - } + if env::var_os(EVM_ARITH_VER_KEY).is_none() { + // Safety: + // - we're early enough in main that nothing else should race + unsafe { + env::set_var( + EVM_ARITH_VER_KEY, + // see build.rs + env!("EVM_ARITHMETIZATION_PACKAGE_VERSION"), + ); } - } + }; let args = cli::Cli::parse(); if let paladin::config::Runtime::InMemory = args.paladin.runtime { diff --git a/zero_bin/leader/src/utils.rs b/zero_bin/leader/src/utils.rs deleted file mode 100644 index e2c7d5f10..000000000 --- a/zero_bin/leader/src/utils.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::fs::File; -use std::io::{BufReader, Read}; -use std::path::Path; - -use anyhow::Result; - -/// Retrieves the version of a specified package from the `Cargo.lock` file. -/// -/// This function attempts to find the version of a package specified by -/// `package_name` by reading and parsing the `Cargo.lock` file. The -/// `Cargo.lock` file is expected to be located one directory level up from the -/// directory specified by the `CARGO_MANIFEST_DIR` environment variable. The -/// path may need adjustment depending on the structure of the project. -/// -/// # Parameters -/// - `package_name`: The name of the package for which the version is being -/// retrieved. -/// -/// # Returns -/// - `Ok(Some(String))`: If the package is found in the `Cargo.lock` file, -/// returns the version of the package. -/// - `Ok(None)`: If the package is not found in the `Cargo.lock` file, or if -/// the `Cargo.lock` file does not exist. -/// - `Err(_)`: If any error occurs during the execution, such as issues with -/// file paths, file access, reading, or parsing the `Cargo.lock` file. -/// -/// # Examples -/// ```no_run -/// let version = get_package_version("my_package"); -/// match version { -/// Ok(Some(ver)) => println!("Found version: {}", ver), -/// Ok(None) => println!("Package not found."), -/// Err(e) => println!("Error occurred: {}", e), -/// } -/// ``` -/// -/// # Errors -/// This function can return an `Err` result if: -/// - There is a problem finding, opening, or reading the `Cargo.lock` file. -/// - There is a failure in parsing the `Cargo.lock` file as TOML. -/// -/// The function uses `?` to propagate errors upwards, so the exact nature of -/// the error will be indicated by the error value returned in the `Err` variant -/// of the `Result`. -pub(crate) fn get_package_version(package_name: &str) -> Result> { - let manifest_dir = env!("CARGO_MANIFEST_DIR"); - let zero_bin_path = Path::new(manifest_dir) - .join("../") // Adjust the path according to your workspace structure - .canonicalize()?; - - let cargo_lock_path = zero_bin_path.join("Cargo.lock"); - let cargo_lock_file = File::open(cargo_lock_path); - if cargo_lock_file.is_err() { - return Ok(None); - } - - let mut cargo_lock_contents = String::new(); - BufReader::new(cargo_lock_file?).read_to_string(&mut cargo_lock_contents)?; - - let lockfile: toml::Value = toml::from_str(&cargo_lock_contents)?; - if let Some(package) = lockfile["package"] - .as_array() - .unwrap() - .iter() - .find(|&p| p["name"].as_str() == Some(package_name)) - { - let version = package["version"].as_str().unwrap(); - return Ok(Some(version.to_string())); - } - - Ok(None) -} diff --git a/zero_bin/rpc/src/lib.rs b/zero_bin/rpc/src/lib.rs index f81674748..fddc19364 100644 --- a/zero_bin/rpc/src/lib.rs +++ b/zero_bin/rpc/src/lib.rs @@ -100,7 +100,7 @@ where std::iter::successors(Some(target_block_number as i128 - 1 + odd_offset), |&it| { Some(it - 1) }) - .take(PREVIOUS_HASHES_COUNT) + .take(PREVIOUS_HASHES_COUNT + 1) .filter(|i| *i >= 0) .collect::>(); let concurrency = previous_block_numbers.len(); @@ -138,6 +138,7 @@ where .into_iter() .flatten() .skip(odd_offset as usize) + .take(PREVIOUS_HASHES_COUNT) .for_each(|(hash, block_num)| { if let (Some(hash), Some(block_num)) = (hash, block_num) { // Most recent previous block hash is expected at the end of the array diff --git a/zero_bin/tools/prove_rpc.sh b/zero_bin/tools/prove_rpc.sh index 9a0700f22..8dd8ea20d 100755 --- a/zero_bin/tools/prove_rpc.sh +++ b/zero_bin/tools/prove_rpc.sh @@ -13,9 +13,8 @@ export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=1 export RUST_LOG=info -# Disable the lld linker for now, as it's causing issues with the linkme package. -# https://github.com/rust-lang/rust/pull/124129 -# https://github.com/dtolnay/linkme/pull/88 +# Script users are running locally, and might benefit from extra perf. +# See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' if [[ $8 == "test_only" ]]; then @@ -29,7 +28,7 @@ if [[ $8 == "test_only" ]]; then export MEMORY_CIRCUIT_SIZE="17..18" else export ARITHMETIC_CIRCUIT_SIZE="16..23" - export BYTE_PACKING_CIRCUIT_SIZE="9..21" + export BYTE_PACKING_CIRCUIT_SIZE="8..21" export CPU_CIRCUIT_SIZE="12..25" export KECCAK_CIRCUIT_SIZE="14..20" export KECCAK_SPONGE_CIRCUIT_SIZE="9..15" diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index e5f5409d0..005ec497a 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -29,9 +29,8 @@ export TOKIO_WORKER_THREADS=$num_procs export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full export RUST_LOG=info -# Disable the lld linker for now, as it's causing issues with the linkme package. -# https://github.com/rust-lang/rust/pull/124129 -# https://github.com/dtolnay/linkme/pull/88 +# Script users are running locally, and might benefit from extra perf. +# See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' INPUT_FILE=$1 @@ -56,17 +55,17 @@ else # These sizes are configured specifically for block 19807080. Don't use this in other scenarios echo "Using specific circuit sizes for witness_b19807080.json" export ARITHMETIC_CIRCUIT_SIZE="16..18" - export BYTE_PACKING_CIRCUIT_SIZE="15..19" + export BYTE_PACKING_CIRCUIT_SIZE="11..15" export CPU_CIRCUIT_SIZE="17..21" export KECCAK_CIRCUIT_SIZE="14..17" export KECCAK_SPONGE_CIRCUIT_SIZE="10..13" export LOGIC_CIRCUIT_SIZE="13..16" - export MEMORY_CIRCUIT_SIZE="20..23" + export MEMORY_CIRCUIT_SIZE="19..23" elif [[ $INPUT_FILE == *"witness_b3_b6"* ]]; then # These sizes are configured specifically for custom blocks 3 to 6. Don't use this in other scenarios echo "Using specific circuit sizes for witness_b3_b6.json" export ARITHMETIC_CIRCUIT_SIZE="16..17" - export BYTE_PACKING_CIRCUIT_SIZE="12..17" + export BYTE_PACKING_CIRCUIT_SIZE="8..14" export CPU_CIRCUIT_SIZE="14..19" export KECCAK_CIRCUIT_SIZE="14..15" export KECCAK_SPONGE_CIRCUIT_SIZE="10..11" @@ -74,7 +73,7 @@ else export MEMORY_CIRCUIT_SIZE="17..21" else export ARITHMETIC_CIRCUIT_SIZE="16..23" - export BYTE_PACKING_CIRCUIT_SIZE="9..21" + export BYTE_PACKING_CIRCUIT_SIZE="8..21" export CPU_CIRCUIT_SIZE="12..25" export KECCAK_CIRCUIT_SIZE="14..20" export KECCAK_SPONGE_CIRCUIT_SIZE="9..15"