Merging Cargo.toml dependencies into workspace Cargo.toml (#339)

Co-authored-by: Brian Ronan <brian.ronan@huggingface.co>

Author: jgodlew

Cargo.lock

@@ -5,20 +5,20 @@ members = [
     "cas_client",
     "cas_object",
     "cas_types",
+    "chunk_cache",
     "data",
+    "deduplication",
     "error_printer",
     "file_utils",
-    "merklehash",
     "mdb_shard",
+    "merklehash",
     "parutils",
+    "progress_tracking",
     "utils",
-    "cas_object",
-    "cas_types",
-    "chunk_cache",
-    "xet_threadpool", 
-    "deduplication", "progress_tracking"]
+    "xet_threadpool"
+]
 
-exclude = ["hf_xet", "chunk_cache_bench", "wasm_xet"]
+exclude = ["chunk_cache_bench", "hf_xet", "wasm_xet"]
 
 [profile.release]
 opt-level = 3
@@ -29,3 +29,78 @@ debug = 1
 inherits = "dev"
 opt-level = 3
 debug = 1
+
+[workspace.dependencies]
+anyhow = "1"
+async-scoped = { version = "0.7", features = ["use-tokio"] }
+async-trait = "0.1"
+base64 = "0.22"
+bincode = "1.3"
+bitflags = { version = "2.9", features = ["serde"] }
+blake3 = "1.5"
+bytes = "1.8"
+chrono = "0.4"
+clap = { version = "4", features = ["derive"] }
+colored = "2"
+countio = { version = "0.2", features = ["futures"] }
+crc32fast = "1.4"
+csv = "1"
+ctor = "0.4"
+dirs = "5.0"
+futures = "0.3"
+futures-io = "0.3"
+futures-util = "0.3"
+gearhash = "0.1"
+getrandom = "0.3"
+half = "2.4"
+heed = "0.11"
+http = "1"
+itertools = "0.14"
+jsonwebtoken = "9.3"
+lazy_static = "1.5"
+len-trait = "0.6"
+libc = "0.2"
+lz4_flex = "0.11"
+mockall = "0.13"
+more-asserts = "0.3"
+once_cell = "1.20"
+openssl = "0.10"
+parking_lot = "0.12"
+pin-project = "1"
+prometheus = "0.14"
+rand = "0.9"
+rand_chacha = "0.9"
+rayon = "1.5"
+regex = "1"
+reqwest = { version = "0.12", features = ["json", "stream"] }
+reqwest-middleware = "0.4"
+reqwest-retry = "0.7"
+rustc-hash = "1.1"
+safe-transmute = "0.11"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+serde_repr = "0.1"
+sha2 = "0.10"
+static_assertions = "1.1"
+tempfile = "3.20"
+thiserror = "2.0"
+tokio = { version = "1.44", features = ["sync"] }
+tokio-retry = "0.3"
+tokio-util = { version = "0.7", features = ["io"] }
+tracing = "0.1"
+ulid = "1.2"
+uuid = "1"
+url = "2.5"
+walkdir = "2"
+whoami = "1"
+
+# windows
+winapi = { version = "0.3", features = ["winerror", "winnt", "handleapi", "processthreadsapi", "securitybaseapi"] }
+
+# dev-deps
+criterion = { version = "0.5", features = ["html_reports"] }
+httpmock = "0.7"
+serial_test = "3"
+tempdir = "0.3"
+tracing-test = { version = "0.2", features = ["no-env-filter"] }
+wiremock = "0.6"

Cargo.toml

@@ -5,43 +5,41 @@ edition = "2021"
 
 [features]
 strict = []
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-cas_types = { version = "0.1.0", path = "../cas_types" }
 cas_object = { path = "../cas_object" }
+cas_types = { path = "../cas_types" }
 chunk_cache = { path = "../chunk_cache" }
+deduplication = { path = "../deduplication" }
 error_printer = { path = "../error_printer" }
 file_utils = { path = "../file_utils" }
-utils = { path = "../utils" }
-merkledb = { path = "../merkledb" }
 mdb_shard = { path = "../mdb_shard" }
+merkledb = { path = "../merkledb" }
 merklehash = { path = "../merklehash" }
-xet_threadpool = { path = "../xet_threadpool" }
-deduplication = { path = "../deduplication" }
 progress_tracking = { path = "../progress_tracking" }
-thiserror = "2.0"
-tokio = { version = "1.44", features = ["sync"] }
-async-trait = "0.1.9"
-anyhow = "1"
-http = "1.1.0"
-tempfile = "3.13.0"
-tracing = "0.1.41"
-bytes = "1"
-itertools = "0.10"
-reqwest = { version = "0.12.7", features = ["json", "stream"] }
-reqwest-middleware = "0.3.3"
-url = "2.5.2"
-reqwest-retry = "0.6.1"
-heed = "0.11"
-futures = "0.3.31"
-derivative = "2.2.0"
-more-asserts = "0.3.1"
-tokio-retry = "0.3.0"
+utils = { path = "../utils" }
+xet_threadpool = { path = "../xet_threadpool" }
+
+anyhow = { workspace = true }
+async-trait = { workspace = true }
+bytes = { workspace = true }
+futures = { workspace = true }
+heed = { workspace = true }
+http = { workspace = true }
+more-asserts = { workspace = true }
+reqwest = { workspace = true }
+reqwest-middleware = { workspace = true }
+reqwest-retry = { workspace = true }
+tempfile = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true }
+tokio-retry = { workspace = true }
+tracing = { workspace = true }
+url = { workspace = true }
 
 [dev-dependencies]
-rand = "0.8.5"
-httpmock = "0.7.0"
-wiremock = "0.6"
-tracing-test = { version = "0.2.5", features = ["no-env-filter"] }
+httpmock = { workspace = true }
+rand = { workspace = true }
+tracing-test = { workspace = true }
+wiremock = { workspace = true }

cas_client/Cargo.toml

@@ -1,12 +1,12 @@
 use std::collections::HashMap;
+use std::fmt;
 use std::sync::{Arc, Mutex, RwLock};
 use std::time::{Duration, Instant};
 
 use cas_object::error::CasObjectError;
 use cas_types::{CASReconstructionFetchInfo, CASReconstructionTerm, ChunkRange, FileRange, HexMerkleHash, Key};
 use chunk_cache::ChunkCache;
 use deduplication::constants::MAX_XORB_BYTES;
-use derivative::Derivative;
 use error_printer::ErrorPrinter;
 use futures::TryStreamExt;
 use http::header::RANGE;
@@ -137,21 +137,32 @@ impl FetchInfo {
 
 /// Helper object containing the structs needed when downloading a term in parallel
 /// during reconstruction.
-#[derive(Derivative)]
-#[derivative(Debug)]
 pub(crate) struct TermDownload {
     pub term: CASReconstructionTerm,
     pub skip_bytes: u64, // number of bytes to skip at the front
     pub take: u64,       // number of bytes to take after skipping bytes,
     // effectively taking [skip_bytes..skip_bytes+take]
     // out of the downloaded range
     pub fetch_info: Arc<FetchInfo>, // utility to get URL to download this term
-    #[derivative(Debug = "ignore")]
     pub chunk_cache: Option<Arc<dyn ChunkCache>>,
     pub client: Arc<ClientWithMiddleware>, // only used for downloading range
     pub range_download_single_flight: RangeDownloadSingleFlight,
 }
 
+impl fmt::Debug for TermDownload {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // print out everything except the chunk cache
+        f.debug_struct("TermDownload")
+            .field("term", &self.term)
+            .field("skip_bytes", &self.skip_bytes)
+            .field("take", &self.take)
+            .field("fetch_info", &self.fetch_info)
+            .field("client", &self.client)
+            .field("range_download_single_flight", &self.range_download_single_flight)
+            .finish()
+    }
+}
+
 #[derive(Debug)]
 pub(crate) struct TermDownloadResult<T> {
     pub data: T,            // download result

cas_client/src/download_utils.rs

@@ -14,31 +14,30 @@ harness = false
 bench = true
 
 [dependencies]
-thiserror = "2.0"
+deduplication = { path = "../deduplication" }
 error_printer = { path = "../error_printer" }
+mdb_shard = { path = "../mdb_shard" }
 merkledb = { path = "../merkledb" }
 merklehash = { path = "../merklehash" }
-mdb_shard = { path = "../mdb_shard" }
 utils = { path = "../utils" }
-parutils = { path = "../parutils" }
-deduplication = { path = "../deduplication" }
-anyhow = "1.0.88"
-tracing = "0.1.40"
-lz4_flex = "0.11.3"
-bytes = "1.7.2"
-rand = "0.9"
-blake3 = "1.5.4"
-futures = { version = "0.3.31" }
-tokio = { version = "1.44", features = ["sync", "time", "rt", "macros"] }
-half = "2.4.1"
-serde = { version = "1.0.129", features = ["derive"] }
-countio = { version = "0.2.19", features = ["futures"] }
-clap = { version = "4.1", features = ["derive"] }
-csv = "1.1"
-more-asserts = "0.3.1"
+
+anyhow = { workspace = true }
+blake3 = { workspace = true }
+bytes = { workspace = true }
+clap = { workspace = true }
+countio = { workspace = true }
+csv = { workspace = true }
+futures = { workspace = true }
+half = { workspace = true }
+lz4_flex = { workspace = true }
+rand = { workspace = true }
+serde = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true, features = ["time", "rt", "macros"] }
+tracing = { workspace = true }
 
 [target.'cfg(not(target_family = "wasm"))'.dependencies]
-tokio-util = { version = "0.7.12", features = ["io"] }
+tokio-util = { workspace = true }
 
 [[bin]]
 path = "src/byte_grouping/compression_stats/collect_compression_stats.rs"

cas_object/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 merklehash = { path = "../merklehash" }
-thiserror = "2.0"
-serde = { version = "1.0.208", features = ["derive"] }
-serde_repr = "0.1.19"
-derivative = "2.2.0"
+
+serde = { workspace = true }
+serde_repr = { workspace = true }
+thiserror = { workspace = true }

cas_types/Cargo.toml

@@ -4,7 +4,6 @@ use std::collections::{HashMap, HashSet};
 use std::marker::PhantomData;
 use std::str::FromStr;
 
-use derivative::Derivative;
 use merklehash::MerkleHash;
 use serde::{Deserialize, Serialize};
 use serde_repr::{Deserialize_repr, Serialize_repr};
@@ -98,16 +97,26 @@ impl From<FileRange> for HttpRange {
 }
 
 // note that the standard PartialOrd/Ord impls will first check `start` then `end`
-#[derive(Derivative, Serialize, Deserialize, Clone, Eq, PartialEq, PartialOrd, Ord, Default, Hash)]
-#[derivative(Debug)]
+#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, PartialOrd, Ord, Default, Hash)]
 pub struct Range<Idx, Kind> {
     pub start: Idx,
     pub end: Idx,
     #[serde(skip)]
-    #[derivative(Debug = "ignore")]
     pub _marker: PhantomData<Kind>,
 }
 
+impl<Idx, _C> fmt::Debug for Range<Idx, _C>
+where
+    Idx: fmt::Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Range")
+            .field("start", &self.start)
+            .field("end", &self.end)
+            .finish()
+    }
+}
+
 impl<Idx, Kind> Range<Idx, Kind> {
     pub fn new(start: Idx, end: Idx) -> Self {
         Self {

cas_types/src/lib.rs

@@ -5,31 +5,30 @@ edition = "2021"
 
 [dependencies]
 cas_types = { path = "../cas_types" }
-merklehash = { path = "../merklehash" }
-thiserror = "2.0"
 error_printer = { path = "../error_printer" }
 file_utils = { path = "../file_utils" }
+merklehash = { path = "../merklehash" }
 utils = { path = "../utils" }
-base64 = "0.22.1"
-tracing = "0.1.40"
-rand = "0.8.5"
-mockall = "0.13.0"
-clap = { version = "4.5.20", optional = true, features = ["derive"] }
-once_cell = "1.20.2"
-crc32fast = "1.4.2"
-log = "0.4.22"
-async-trait = "0.1.88"
-tokio = { version = "1.44", features = ["sync"] }
+
+async-trait = { workspace = true }
+base64 = { workspace = true }
+clap = { workspace = true, optional = true }
+crc32fast = { workspace = true }
+mockall = { workspace = true }
+once_cell = { workspace = true }
+rand = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }
 
 [dev-dependencies]
-tokio = { version = "1.44", features = ["full"] }
-tempdir = "0.3.7"
+tempdir = { workspace = true }
+tokio = { workspace = true, features = ["full"] }
 
 [[bin]]
 name = "cache_analysis"
 path = "./src/bin/analysis.rs"
 required-features = ["analysis"]
 
-
 [features]
 analysis = ["dep:clap"]

chunk_cache/Cargo.toml

@@ -113,7 +113,7 @@ impl CacheState {
             error!("cache random_item for eviction: no items in cache");
             return None;
         }
-        let random_item = rand::random::<usize>() % self.num_items;
+        let random_item = rand::random::<u32>() as usize % self.num_items;
         let mut count = 0;
         for (key, items) in self.inner.iter() {
             if random_item < count + items.len() {
@@ -1003,7 +1003,7 @@ mod tests {
 
         // not matching
         let (_cache_root, cache, key, range, mut offsets, data) = setup().await;
-        offsets[1] = offsets[1] + 1;
+        offsets[1] += 1;
         assert!(cache.put(&key, &range, &offsets, &data).await.is_err());
 
         // bad data
@@ -1013,7 +1013,7 @@ mod tests {
 
         // data changed
         let (_cache_root, cache, key, range, offsets, mut data) = setup().await;
-        data[0] = data[0] + 1;
+        data[0] += 1;
         assert!(cache.put(&key, &range, &offsets, &data).await.is_err());
     }