Skip to content

Commit d472d60

Browse files
authored
chore: add compile time flags to disable rayon (#240)
* maybe-rayon package * refactor code * remove rayon from dep * fix features * remove default * conditionally compile the DasContext constructor * use multithread and singlethreaded * turn on multithreading for c_kzg * add feature flags
1 parent c00bbe0 commit d472d60

File tree

15 files changed

+202
-23
lines changed

15 files changed

+202
-23
lines changed

Cargo.toml

+2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ members = [
77
"bindings/nim/rust_code",
88
"bindings/csharp/rust_code",
99
"eip7594",
10+
"maybe_rayon",
1011
"cryptography/bls12_381",
1112
"cryptography/kzg_multi_open",
1213
"cryptography/polynomial",
@@ -31,6 +32,7 @@ bls12_381 = { package = "crate_crypto_internal_eth_kzg_bls12_381", version = "0.
3132
polynomial = { package = "crate_crypto_internal_eth_kzg_polynomial", version = "0.4.1", path = "cryptography/polynomial" }
3233
erasure_codes = { package = "crate_crypto_internal_eth_kzg_erasure_codes", version = "0.4.1", path = "cryptography/erasure_codes" }
3334
rust_eth_kzg = { version = "0.4.1", path = "eip7594" }
35+
maybe_rayon = { package = "crate_crypto_internal_eth_kzg_maybe_rayon", version = "0.4.1", path = "maybe_rayon" }
3436
kzg_multi_open = { package = "crate_crypto_kzg_multi_open_fk20", version = "0.4.1", path = "cryptography/kzg_multi_open" }
3537
c_eth_kzg = { version = "0.4.1", path = "bindings/c" }
3638
hex = "0.4.3"

bindings/c/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ crate-type = ["staticlib", "cdylib", "rlib"]
1313

1414
[dependencies]
1515
libc = "0.2.2"
16-
rust_eth_kzg = { workspace = true }
16+
rust_eth_kzg = { workspace = true, features = ["multithreaded"] }
1717

1818
[build-dependencies]
1919
cbindgen = "0.26.0"

bindings/node/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ napi = { version = "2.12.2", default-features = false, features = [
1717
"async",
1818
] }
1919
napi-derive = "2.12.2"
20-
rust_eth_kzg = { workspace = true }
20+
rust_eth_kzg = { workspace = true, features = ["multithreaded"] }
2121

2222
[build-dependencies]
2323
napi-build = "2.0.1"

cryptography/bls12_381/Cargo.toml

+3
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ subtle = { version = ">=2.5.0, <3.0" }
3131
criterion = "0.5.1"
3232
rand = "0.8.4"
3333

34+
[features]
35+
blst-no-threads = ["blst/no-threads"]
36+
3437
[[bench]]
3538
name = "benchmark"
3639
harness = false

cryptography/kzg_multi_open/Cargo.toml

+5-1
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,18 @@ repository = { workspace = true }
1313
[dependencies]
1414
bls12_381 = { workspace = true }
1515
polynomial = { workspace = true }
16+
maybe_rayon = { workspace = true }
1617
hex = { workspace = true }
17-
rayon = { workspace = true }
1818
sha2 = "0.10.8"
1919

2020
[dev-dependencies]
2121
criterion = "0.5.1"
2222
rand = "0.8.4"
2323

24+
[features]
25+
singlethreaded = ["bls12_381/blst-no-threads"]
26+
multithreaded = ["maybe_rayon/multithreaded"]
27+
2428
[[bench]]
2529
name = "benchmark"
2630
harness = false

cryptography/kzg_multi_open/src/fk20/batch_toeplitz.rs

+9-8
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ use bls12_381::{
33
fixed_base_msm::{FixedBaseMSM, UsePrecomp},
44
g1_batch_normalize, G1Point, G1Projective,
55
};
6+
use maybe_rayon::prelude::*;
67
use polynomial::domain::Domain;
7-
use rayon::prelude::*;
88

99
/// BatchToeplitzMatrixVecMul allows one to compute multiple matrix vector multiplications
1010
/// and sum them together.
@@ -43,7 +43,7 @@ impl BatchToeplitzMatrixVecMul {
4343

4444
// Precompute the FFT of the vectors, since they do not change per matrix-vector multiplication
4545
let vectors: Vec<Vec<G1Point>> = vectors
46-
.into_par_iter()
46+
.maybe_par_iter()
4747
.map(|vector| {
4848
let vector_projective = vector
4949
.iter()
@@ -61,7 +61,7 @@ impl BatchToeplitzMatrixVecMul {
6161
//
6262
// This is a trade-off between storage and computation, where storage grows exponentially.
6363
let precomputed_table: Vec<_> = transposed_msm_vectors
64-
.into_par_iter()
64+
.maybe_into_par_iter()
6565
.map(|v| FixedBaseMSM::new(v, use_precomp))
6666
.collect();
6767

@@ -87,22 +87,23 @@ impl BatchToeplitzMatrixVecMul {
8787
);
8888

8989
// Embed Toeplitz matrices into circulant matrices
90-
let circulant_matrices = matrices.into_iter().map(CirculantMatrix::from_toeplitz);
90+
let circulant_matrices = matrices
91+
.maybe_into_par_iter()
92+
.map(CirculantMatrix::from_toeplitz);
9193

9294
// Perform circulant matrix-vector multiplication between all of the matrices and vectors
9395
// and sum them together.
9496
//
9597
// Transpose the circulant matrices so that we convert a group of hadamard products into a group of
9698
// inner products.
9799
let col_ffts: Vec<_> = circulant_matrices
98-
.into_iter()
100+
.maybe_into_par_iter()
99101
.map(|matrix| self.circulant_domain.fft_scalars(matrix.row))
100102
.collect();
101103
let msm_scalars = transpose(col_ffts);
102104

103-
let result: Vec<_> = self
104-
.precomputed_fft_vectors
105-
.iter()
105+
let result: Vec<_> = (&self.precomputed_fft_vectors)
106+
.maybe_par_iter()
106107
.zip(msm_scalars)
107108
.map(|(points, scalars)| points.msm(scalars))
108109
.collect();

eip7594/Cargo.toml

+6-1
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,14 @@ kzg_multi_open = { workspace = true }
1313
bls12_381 = { workspace = true }
1414
hex = { workspace = true }
1515
erasure_codes = { workspace = true }
16-
rayon = { workspace = true }
16+
rayon = { workspace = true, optional = true }
1717
serde = { version = "1", features = ["derive"] }
1818
serde_json = "1"
1919

20+
[features]
21+
singlethreaded = ["rayon", "kzg_multi_open/singlethreaded"]
22+
multithreaded = ["rayon", "kzg_multi_open/multithreaded"]
23+
2024
[dev-dependencies]
2125
criterion = "0.5.1"
2226
rand = "0.8.4"
@@ -28,3 +32,4 @@ serde_yaml = "0.9.34"
2832
[[bench]]
2933
name = "benchmark"
3034
harness = false
35+
# required-features = ["multithreaded"]

eip7594/src/lib.rs

+41-5
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
1+
#[cfg(all(feature = "singlethreaded", feature = "multithreaded"))]
2+
compile_error!("feature_a and feature_b cannot be enabled simultaneously");
3+
14
pub mod constants;
25
mod errors;
36
mod prover;
47
mod serialization;
58
mod trusted_setup;
69
mod verifier;
10+
#[macro_use]
11+
pub(crate) mod macros;
712

813
pub use bls12_381::fixed_base_msm::UsePrecomp;
914
// Exported types
@@ -54,9 +59,12 @@ pub type CellIndex = kzg_multi_open::CosetIndex;
5459

5560
use constants::{BYTES_PER_BLOB, BYTES_PER_CELL, BYTES_PER_COMMITMENT};
5661
use prover::ProverContext;
62+
use verifier::VerifierContext;
63+
64+
#[cfg(feature = "multithreaded")]
5765
use rayon::ThreadPool;
66+
#[cfg(feature = "multithreaded")]
5867
use std::sync::Arc;
59-
use verifier::VerifierContext;
6068

6169
/// ThreadCount indicates whether we want to use a single thread or multiple threads
6270
#[derive(Debug, Copy, Clone)]
@@ -65,19 +73,23 @@ pub enum ThreadCount {
6573
Single,
6674
/// Initializes the threadpool with the number of threads
6775
/// denoted by this enum variant.
76+
#[cfg(feature = "multithreaded")]
6877
Multi(usize),
6978
/// Initializes the threadpool with a sensible default number of
7079
/// threads. This is currently set to `RAYON_NUM_THREADS`.
80+
#[cfg(feature = "multithreaded")]
7181
SensibleDefault,
7282
}
7383

7484
impl From<ThreadCount> for usize {
7585
fn from(value: ThreadCount) -> Self {
7686
match value {
7787
ThreadCount::Single => 1,
88+
#[cfg(feature = "multithreaded")]
7889
ThreadCount::Multi(num_threads) => num_threads,
7990
// Setting this to `0` will tell ThreadPool to use
8091
// `RAYON_NUM_THREADS`.
92+
#[cfg(feature = "multithreaded")]
8193
ThreadCount::SensibleDefault => 0,
8294
}
8395
}
@@ -86,29 +98,37 @@ impl From<ThreadCount> for usize {
8698
/// The context that will be used to create and verify opening proofs.
8799
#[derive(Debug)]
88100
pub struct DASContext {
101+
#[cfg(feature = "multithreaded")]
89102
thread_pool: Arc<ThreadPool>,
90103
pub prover_ctx: ProverContext,
91104
pub verifier_ctx: VerifierContext,
92105
}
93106

107+
#[cfg(feature = "multithreaded")]
94108
impl Default for DASContext {
95109
fn default() -> Self {
96110
let trusted_setup = TrustedSetup::default();
97111
const DEFAULT_NUM_THREADS: ThreadCount = ThreadCount::Single;
98112
DASContext::with_threads(&trusted_setup, DEFAULT_NUM_THREADS, UsePrecomp::No)
99113
}
100114
}
115+
#[cfg(not(feature = "multithreaded"))]
116+
impl Default for DASContext {
117+
fn default() -> Self {
118+
let trusted_setup = TrustedSetup::default();
119+
120+
DASContext::new(&trusted_setup, UsePrecomp::No)
121+
}
122+
}
101123

102124
impl DASContext {
125+
#[cfg(feature = "multithreaded")]
103126
pub fn with_threads(
104127
trusted_setup: &TrustedSetup,
105128
num_threads: ThreadCount,
106-
// This parameter indicates whether we should allocate memory
107-
// in order to speed up proof creation. Heuristics show that
108-
// if pre-computations are desired, one should set the
109-
// width value to `8` for optimal storage and performance tradeoffs.
110129
use_precomp: UsePrecomp,
111130
) -> Self {
131+
#[cfg(feature = "multithreaded")]
112132
let thread_pool = std::sync::Arc::new(
113133
rayon::ThreadPoolBuilder::new()
114134
.num_threads(num_threads.into())
@@ -117,12 +137,28 @@ impl DASContext {
117137
);
118138

119139
DASContext {
140+
#[cfg(feature = "multithreaded")]
120141
thread_pool,
121142
prover_ctx: ProverContext::new(trusted_setup, use_precomp),
122143
verifier_ctx: VerifierContext::new(trusted_setup),
123144
}
124145
}
125146

147+
#[cfg(not(feature = "multithreaded"))]
148+
pub fn new(
149+
trusted_setup: &TrustedSetup,
150+
// This parameter indicates whether we should allocate memory
151+
// in order to speed up proof creation. Heuristics show that
152+
// if pre-computations are desired, one should set the
153+
// width value to `8` for optimal storage and performance tradeoffs.
154+
use_precomp: UsePrecomp,
155+
) -> Self {
156+
DASContext {
157+
prover_ctx: ProverContext::new(trusted_setup, use_precomp),
158+
verifier_ctx: VerifierContext::new(trusted_setup),
159+
}
160+
}
161+
126162
pub fn prover_ctx(&self) -> &ProverContext {
127163
&self.prover_ctx
128164
}

eip7594/src/macros.rs

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#[macro_export]
2+
macro_rules! with_optional_threadpool {
3+
($self:expr, $body:expr) => {{
4+
#[cfg(feature = "multithreaded")]
5+
{
6+
$self.thread_pool.install(|| $body)
7+
}
8+
#[cfg(not(feature = "multithreaded"))]
9+
{
10+
$body
11+
}
12+
}};
13+
}

eip7594/src/prover.rs

+5-4
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ use crate::{
1414
deserialize_blob_to_scalars, serialize_cells_and_proofs, serialize_g1_compressed,
1515
},
1616
trusted_setup::TrustedSetup,
17-
BlobRef, Cell, CellIndex, CellRef, DASContext, KZGCommitment, KZGProof,
17+
with_optional_threadpool, BlobRef, Cell, CellIndex, CellRef, DASContext, KZGCommitment,
18+
KZGProof,
1819
};
1920

2021
/// Context object that is used to call functions in the prover API.
@@ -64,7 +65,7 @@ impl DASContext {
6465
///
6566
/// The matching function in the specs is: https://github.com/ethereum/consensus-specs/blob/13ac373a2c284dc66b48ddd2ef0a10537e4e0de6/specs/deneb/polynomial-commitments.md#blob_to_kzg_commitment
6667
pub fn blob_to_kzg_commitment(&self, blob: BlobRef) -> Result<KZGCommitment, Error> {
67-
self.thread_pool.install(|| {
68+
with_optional_threadpool!(self, {
6869
// Deserialize the blob into scalars.
6970
let scalars = deserialize_blob_to_scalars(blob)?;
7071

@@ -86,7 +87,7 @@ impl DASContext {
8687
&self,
8788
blob: BlobRef,
8889
) -> Result<([Cell; CELLS_PER_EXT_BLOB], [KZGProof; CELLS_PER_EXT_BLOB]), Error> {
89-
self.thread_pool.install(|| {
90+
with_optional_threadpool!(self, {
9091
// Deserialization
9192
//
9293
let scalars = deserialize_blob_to_scalars(blob)?;
@@ -116,7 +117,7 @@ impl DASContext {
116117
cell_indices: Vec<CellIndex>,
117118
cells: Vec<CellRef>,
118119
) -> Result<([Cell; CELLS_PER_EXT_BLOB], [KZGProof; CELLS_PER_EXT_BLOB]), Error> {
119-
self.thread_pool.install(|| {
120+
with_optional_threadpool!(self, {
120121
// Recover polynomial
121122
//
122123
let poly_coeff = self.recover_polynomial_coeff(cell_indices, cells)?;

eip7594/src/verifier.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use crate::{
99
errors::Error,
1010
serialization::{deserialize_cells, deserialize_compressed_g1_points},
1111
trusted_setup::TrustedSetup,
12-
Bytes48Ref, CellIndex, CellRef, DASContext,
12+
with_optional_threadpool, Bytes48Ref, CellIndex, CellRef, DASContext,
1313
};
1414
use bls12_381::Scalar;
1515
use erasure_codes::{BlockErasureIndices, ReedSolomon};
@@ -103,7 +103,7 @@ impl DASContext {
103103
cells: Vec<CellRef>,
104104
proofs_bytes: Vec<Bytes48Ref>,
105105
) -> Result<(), Error> {
106-
self.thread_pool.install(|| {
106+
with_optional_threadpool!(self, {
107107
let (deduplicated_commitments, row_indices) = deduplicate_with_indices(commitments);
108108
// Validation
109109
//

maybe_rayon/Cargo.toml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[package]
2+
name = "crate_crypto_internal_eth_kzg_maybe_rayon"
3+
description = "This crate provides an implementation of a wrapper around the rayon crate"
4+
version = { workspace = true }
5+
authors = { workspace = true }
6+
edition = { workspace = true }
7+
license = { workspace = true }
8+
rust-version = { workspace = true }
9+
repository = { workspace = true }
10+
11+
[dependencies]
12+
rayon = { workspace = true, optional = true }
13+
14+
[features]
15+
multithreaded = ["rayon"]

maybe_rayon/src/lib.rs

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#[cfg(feature = "multithreaded")]
2+
mod multi_threaded;
3+
#[cfg(not(feature = "multithreaded"))]
4+
mod single_threaded;
5+
6+
#[cfg(feature = "multithreaded")]
7+
pub use multi_threaded::*;
8+
#[cfg(not(feature = "multithreaded"))]
9+
pub use single_threaded::*;
10+
11+
pub mod prelude {
12+
pub use crate::MaybeParallelRefExt;
13+
pub use crate::MaybeParallelRefMutExt;
14+
pub use crate::*;
15+
#[cfg(feature = "multithreaded")]
16+
pub use rayon::prelude::*;
17+
}

maybe_rayon/src/multi_threaded.rs

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
pub use rayon::iter::IntoParallelIterator;
2+
pub use rayon::iter::IntoParallelRefIterator;
3+
pub use rayon::iter::IntoParallelRefMutIterator;
4+
pub use rayon::iter::ParallelIterator;
5+
6+
pub trait MaybeParallelExt: IntoParallelIterator {
7+
fn maybe_into_par_iter(self) -> <Self as IntoParallelIterator>::Iter
8+
where
9+
Self: Sized,
10+
{
11+
self.into_par_iter()
12+
}
13+
}
14+
15+
pub trait MaybeParallelRefExt: for<'a> IntoParallelRefIterator<'a> {
16+
fn maybe_par_iter(&self) -> <Self as IntoParallelRefIterator>::Iter {
17+
self.par_iter()
18+
}
19+
}
20+
21+
pub trait MaybeParallelRefMutExt: for<'a> IntoParallelRefMutIterator<'a> {
22+
fn maybe_par_iter_mut(&mut self) -> <Self as IntoParallelRefMutIterator>::Iter {
23+
self.par_iter_mut()
24+
}
25+
}
26+
27+
impl<T: IntoParallelIterator> MaybeParallelExt for T {}
28+
impl<T: for<'a> IntoParallelRefIterator<'a>> MaybeParallelRefExt for T {}
29+
impl<T: for<'a> IntoParallelRefMutIterator<'a>> MaybeParallelRefMutExt for T {}

0 commit comments

Comments
 (0)