remove need for serde_json

Author: marvin-j97

.github/workflows/test.yml

@@ -13,7 +13,7 @@ env:
 
 jobs:
   test:
-    timeout-minutes: 20
+    timeout-minutes: 10
     strategy:
       matrix:
         rust_version:
@@ -38,8 +38,10 @@ jobs:
         uses: Swatinem/rust-cache@v2
         with:
           prefix-key: ${{ runner.os }}-cargo
+      - name: Install cargo-all-features
+        run: cargo install cargo-all-features
       - name: Build
-        run: cargo build -v
+        run: cargo build-all-features
       - name: Format
         run: cargo fmt --all -- --check
       - name: Clippy
@@ -49,7 +51,7 @@ jobs:
         env:
           RUST_LOG: debug
   cross:
-    timeout-minutes: 20
+    timeout-minutes: 10
     name: cross
     strategy:
       matrix:

Cargo.toml

@@ -21,6 +21,7 @@ path = "src/lib.rs"
 [features]
 default = ["lz4"]
 lz4 = ["dep:lz4_flex"]
+serde = ["dep:serde"]
 
 [dependencies]
 byteorder = "1.5.0"
@@ -31,12 +32,11 @@ min-max-heap = "1.3.0"
 path-absolutize = "3.1.1"
 quick_cache = "0.4.1"
 rand = "0.8.5"
-serde = { version = "1.0.197", default-features = false, features = [
+serde = { version = "1.0.197", optional = true, features = [
   "alloc",
-  "rc",
   "derive",
+  "rc",
 ] }
-serde_json = { version = "1.0.114" }
 tempfile = "3.10.0"
 
 [dev-dependencies]

src/error.rs

@@ -8,9 +8,9 @@ pub enum Error {
 
     /// Invalid data format version
     InvalidVersion(Option<Version>),
-
-    /// CRC check failed
-    CrcMismatch,
+    // TODO:
+    // /// CRC check failed
+    // CrcMismatch,
 }
 
 impl std::fmt::Display for Error {

src/handle.rs

@@ -1,10 +1,10 @@
 use crate::id::SegmentId;
-use serde::{Deserialize, Serialize};
 use std::hash::Hash;
 
 /// A value handle points into the value log.
 #[allow(clippy::module_name_repetitions)]
-#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
 pub struct ValueHandle {
     /// Segment ID
     pub segment_id: SegmentId,

src/manifest.rs

@@ -1,14 +1,16 @@
 use crate::{id::SegmentId, segment::stats::Stats, Segment, SegmentWriter as MultiWriter};
+use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
 use std::{
     collections::HashMap,
     fs::File,
-    io::Write,
+    io::{Cursor, Write},
     path::{Path, PathBuf},
     sync::{atomic::AtomicU64, Arc, RwLock},
 };
 
 pub const VLOG_MARKER: &str = ".vlog";
 pub const SEGMENTS_FOLDER: &str = "segments";
+const MANIFEST_FILE: &str = "vlog_manifest";
 
 /// Atomically rewrites a file
 fn rewrite_atomic<P: AsRef<Path>>(path: P, content: &[u8]) -> std::io::Result<()> {
@@ -53,8 +55,6 @@ impl SegmentManifest {
         folder: P,
         registered_ids: &[u64],
     ) -> crate::Result<()> {
-        // TODO:
-
         for dirent in std::fs::read_dir(folder)? {
             let dirent = dirent?;
 
@@ -76,14 +76,32 @@ impl SegmentManifest {
         Ok(())
     }
 
+    /// Parses segment IDs from manifest file
+    fn load_ids_from_disk<P: AsRef<Path>>(path: P) -> crate::Result<Vec<SegmentId>> {
+        let path = path.as_ref();
+        log::debug!("Loading manifest from {}", path.display());
+
+        let bytes = std::fs::read(path)?;
+
+        let mut ids = vec![];
+
+        let mut cursor = Cursor::new(bytes);
+
+        let cnt = cursor.read_u64::<BigEndian>()?;
+
+        for _ in 0..cnt {
+            ids.push(cursor.read_u64::<BigEndian>()?);
+        }
+
+        Ok(ids)
+    }
+
+    /// Recovers a value log from disk
     pub(crate) fn recover<P: AsRef<Path>>(folder: P) -> crate::Result<Self> {
         let folder = folder.as_ref();
-        let path = folder.join("segments.json");
-
-        log::debug!("Loading value log manifest from {}", path.display());
+        let path = folder.join(MANIFEST_FILE);
 
-        let str = std::fs::read_to_string(&path)?;
-        let ids: Vec<u64> = serde_json::from_str(&str).expect("deserialize error");
+        let ids = Self::load_ids_from_disk(&path)?;
 
         let segments_folder = folder.join(SEGMENTS_FOLDER);
         Self::remove_unfinished_segments(&segments_folder, &ids)?;
@@ -112,7 +130,7 @@ impl SegmentManifest {
     }
 
     pub(crate) fn create_new<P: AsRef<Path>>(folder: P) -> crate::Result<Self> {
-        let path = folder.as_ref().join("segments.json");
+        let path = folder.as_ref().join(MANIFEST_FILE);
 
         let m = Self(Arc::new(SegmentManifestInner {
             path,
@@ -159,10 +177,16 @@ impl SegmentManifest {
         let path = path.as_ref();
         log::trace!("Writing segment manifest to {}", path.display());
 
-        // NOTE: Serialization can't fail here
-        #[allow(clippy::expect_used)]
-        let json = serde_json::to_string_pretty(&segment_ids).expect("should serialize");
-        rewrite_atomic(path, json.as_bytes())?;
+        let mut bytes = Vec::new();
+
+        let cnt = segment_ids.len() as u64;
+        bytes.write_u64::<BigEndian>(cnt)?;
+
+        for id in segment_ids {
+            bytes.write_u64::<BigEndian>(*id)?;
+        }
+
+        rewrite_atomic(path, &bytes)?;
 
         Ok(())
     }

src/segment/stats.rs

@@ -1,7 +1,7 @@
-use serde::{Deserialize, Serialize};
 use std::sync::atomic::AtomicU64;
 
-#[derive(Debug, Default, Deserialize, Serialize)]
+#[derive(Debug, Default)]
+#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
 pub struct Stats {
     pub(crate) item_count: AtomicU64,
     pub(crate) stale_items: AtomicU64,