Skip to content

Commit

Permalink
feat: add primitives for supporting versioning (#11)
Browse files Browse the repository at this point in the history 
- remove seed deserializer
  • Loading branch information
@appcypher
appcypher authored Oct 20, 2024
1 parent 83eee43 commit b13fb99
Show file tree
Hide file tree
Showing 10 changed files with 627 additions and 180 deletions.
17 changes: 15 additions & 2 deletions monocore/lib/error.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
use std::{error::Error, fmt::Display};
use std::{
error::Error,
fmt::{self, Display},
};
use thiserror::Error;

use crate::oci::distribution::DockerRegistryResponseError;
Expand Down Expand Up @@ -117,6 +120,16 @@ impl MonocoreError {
}
}

impl AnyError {
/// Downcasts the error to a `T`.
pub fn downcast<T>(&self) -> Option<&T>
where
T: Display + fmt::Debug + Send + Sync + 'static,
{
self.error.downcast_ref::<T>()
}
}

//--------------------------------------------------------------------------------------------------
// Functions
//--------------------------------------------------------------------------------------------------
Expand All @@ -138,7 +151,7 @@ impl PartialEq for AnyError {
}

impl Display for AnyError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.error)
}
}
Expand Down
8 changes: 7 additions & 1 deletion monofs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,20 @@
</p>
</div>

**`monofs`** is a powerful, distributed filesystem library designed for AI-driven sandboxed environments. It provides a simple and intuitive API for managing files and directories in a content-addressed storage system.
**`monofs`** is a powerful, distributed filesystem designed for distributed workloads. It provides a simple and intuitive API for managing files and directories in a content-addressed storage system.

> [!WARNING]
> This project is in early development and is not yet ready for production use.
##

## Features

- Content-addressed storage
- Immutable data structures with copy-on-write semantics
- Support for files, directories, and symbolic links
- Asynchronous API for efficient I/O operations
- Versioning support for tracking file and directory history

## Usage

Expand Down
203 changes: 150 additions & 53 deletions monofs/lib/filesystem/dir/dir.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,13 @@ use std::{
collections::{BTreeMap, HashMap},
fmt::{self, Debug},
str::FromStr,
sync::Arc,
sync::{Arc, OnceLock},
};

use monoutils_store::{
ipld::cid::Cid, IpldReferences, IpldStore, Storable, StoreError, StoreResult,
};
use serde::{
de::{self, DeserializeSeed},
Deserialize, Deserializer, Serialize,
};
use serde::{Deserialize, Serialize};

use crate::filesystem::{
kind::EntityType, Entity, EntityCidLink, File, FsError, FsResult, Link, Metadata, SoftLink,
Expand Down Expand Up @@ -42,14 +39,22 @@ pub(super) struct DirInner<S>
where
S: IpldStore,
{
/// The CID of the directory when it is initially loaded from the store.
///
/// It is not initialized if the directory was not loaded from the store.
initial_load_cid: OnceLock<Cid>,

/// The CID of the previous version of the directory if there is one.
previous: Option<Cid>,

/// Directory metadata.
pub(crate) metadata: Metadata,
metadata: Metadata,

/// The store used to persist blocks in the directory.
pub(crate) store: S,
store: S,

/// The entries in the directory.
pub(crate) entries: HashMap<Utf8UnixPathSegment, EntityCidLink<S>>,
entries: HashMap<Utf8UnixPathSegment, EntityCidLink<S>>,
}

//--------------------------------------------------------------------------------------------------
Expand All @@ -60,10 +65,7 @@ where
pub(crate) struct DirSerializable {
metadata: Metadata,
entries: BTreeMap<String, Cid>,
}

pub(crate) struct DirDeserializeSeed<S> {
pub(crate) store: S,
previous: Option<Cid>,
}

//--------------------------------------------------------------------------------------------------
Expand All @@ -76,7 +78,7 @@ where
{
/// Creates a new directory with the given store.
///
/// # Examples
/// ## Examples
///
/// ```
/// use monofs::filesystem::Dir;
Expand All @@ -90,6 +92,8 @@ where
pub fn new(store: S) -> Self {
Self {
inner: Arc::new(DirInner {
initial_load_cid: OnceLock::new(),
previous: None,
metadata: Metadata::new(EntityType::Dir),
entries: HashMap::new(),
store,
Expand All @@ -99,7 +103,7 @@ where

/// Checks if an [`EntityCidLink`] with the given name exists in the directory.
///
/// # Examples
/// ## Examples
///
/// ```
/// use monofs::filesystem::{Dir, Utf8UnixPathSegment};
Expand All @@ -126,9 +130,82 @@ where
Ok(self.inner.entries.contains_key(&name))
}

/// Returns the CID of the directory when it was initially loaded from the store.
///
/// It returns `None` if the directory was not loaded from the store.
///
/// ## Examples
///
/// ```
/// use monofs::filesystem::Dir;
/// use monoutils_store::{MemoryStore, Storable};
/// use monoutils_store::ipld::cid::Cid;
///
/// # #[tokio::main]
/// # async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let store = MemoryStore::default();
/// let dir = Dir::new(store.clone());
///
/// // Initially, the CID is not set
/// assert!(dir.get_initial_load_cid().is_none());
///
/// // Store the directory
/// let stored_cid = dir.store().await?;
///
/// // Load the directory
/// let loaded_dir = Dir::load(&stored_cid, store).await?;
///
/// // Now the initial load CID is set
/// assert_eq!(loaded_dir.get_initial_load_cid(), Some(&stored_cid));
/// # Ok(())
/// # }
/// ```
pub fn get_initial_load_cid(&self) -> Option<&Cid> {
self.inner.initial_load_cid.get()
}

/// Returns the CID of the previous version of the directory if there is one.
///
/// ## Examples
///
/// ```
/// use monofs::filesystem::Dir;
/// use monoutils_store::{MemoryStore, Storable};
/// use monoutils_store::ipld::cid::Cid;
///
/// # #[tokio::main]
/// # async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let store = MemoryStore::default();
/// let mut dir = Dir::new(store.clone());
///
/// // Initially, there's no previous version
/// assert!(dir.get_previous().is_none());
///
/// // Store the directory
/// let first_cid = dir.store().await?;
///
/// // Load the directory and create a new version
/// let mut loaded_dir = Dir::load(&first_cid, store.clone()).await?;
/// loaded_dir.put_entry("new_file", Cid::default().into())?;
///
/// // Store the new version
/// let second_cid = loaded_dir.store().await?;
///
/// // Load the new version
/// let new_version = Dir::load(&second_cid, store).await?;
///
/// // Now the previous CID is set
/// assert_eq!(new_version.get_previous(), Some(&first_cid));
/// # Ok(())
/// # }
/// ```
pub fn get_previous(&self) -> Option<&Cid> {
self.inner.previous.as_ref()
}

/// Adds a [`EntityCidLink`] and its associated name in the directory's entries.
///
/// # Examples
/// ## Examples
///
/// ```
/// use monofs::filesystem::{Dir, Utf8UnixPathSegment};
Expand Down Expand Up @@ -194,7 +271,7 @@ where

/// Gets the [`EntityCidLink`] with the given name from the directory's entries.
///
/// # Examples
/// ## Examples
///
/// ```
/// use monofs::filesystem::{Dir, Utf8UnixPathSegment};
Expand Down Expand Up @@ -355,7 +432,7 @@ where

/// Returns `true` if the directory is empty.
///
/// # Examples
/// ## Examples
///
/// ```
/// use monofs::filesystem::{Dir, Utf8UnixPathSegment};
Expand All @@ -382,18 +459,12 @@ where
self.inner.entries.is_empty()
}

/// Deserializes to a `Dir` using an arbitrary deserializer and store.
pub fn deserialize_with<'de>(
deserializer: impl Deserializer<'de, Error: Into<FsError>>,
/// Tries to create a new `Dir` from a serializable representation.
pub(crate) fn from_serializable(
serializable: DirSerializable,
store: S,
load_cid: Cid,
) -> FsResult<Self> {
DirDeserializeSeed::new(store)
.deserialize(deserializer)
.map_err(Into::into)
}

/// Tries to create a new `Dir` from a serializable representation.
pub(crate) fn try_from_serializable(serializable: DirSerializable, store: S) -> FsResult<Self> {
let entries: HashMap<_, _> = serializable
.entries
.into_iter()
Expand All @@ -402,6 +473,8 @@ where

Ok(Dir {
inner: Arc::new(DirInner {
initial_load_cid: OnceLock::from(load_cid),
previous: serializable.previous,
metadata: serializable.metadata,
store,
entries,
Expand All @@ -410,16 +483,6 @@ where
}
}

//--------------------------------------------------------------------------------------------------
// Methods: DirDeserializeSeed
//--------------------------------------------------------------------------------------------------

impl<S> DirDeserializeSeed<S> {
fn new(store: S) -> Self {
Self { store }
}
}

//--------------------------------------------------------------------------------------------------
// Trait Implementations
//--------------------------------------------------------------------------------------------------
Expand All @@ -439,6 +502,7 @@ where

let serializable = DirSerializable {
metadata: self.inner.metadata.clone(),
previous: self.inner.initial_load_cid.get().cloned(),
entries,
};

Expand All @@ -447,7 +511,7 @@ where

async fn load(cid: &Cid, store: S) -> StoreResult<Self> {
let serializable: DirSerializable = store.get_node(cid).await?;
Dir::try_from_serializable(serializable, store).map_err(StoreError::custom)
Dir::from_serializable(serializable, store, *cid).map_err(StoreError::custom)
}
}

Expand All @@ -469,21 +533,6 @@ where
}
}

impl<'de, S> DeserializeSeed<'de> for DirDeserializeSeed<S>
where
S: IpldStore,
{
type Value = Dir<S>;

fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
where
D: serde::Deserializer<'de>,
{
let serializable = DirSerializable::deserialize(deserializer)?;
Dir::try_from_serializable(serializable, self.store).map_err(de::Error::custom)
}
}

impl IpldReferences for DirSerializable {
fn get_references<'a>(&'a self) -> Box<dyn Iterator<Item = &'a Cid> + Send + 'a> {
Box::new(self.entries.values())
Expand Down Expand Up @@ -665,4 +714,52 @@ mod tests {

Ok(())
}

#[tokio::test]
async fn test_dir_get_initial_load_cid() -> anyhow::Result<()> {
let store = MemoryStore::default();
let dir = Dir::new(store.clone());

// Initially, the CID is not set
assert!(dir.get_initial_load_cid().is_none());

// Store the directory
let stored_cid = dir.store().await?;

// Load the directory
let loaded_dir = Dir::load(&stored_cid, store).await?;

// Now the initial load CID is set
assert_eq!(loaded_dir.get_initial_load_cid(), Some(&stored_cid));

Ok(())
}

#[tokio::test]
async fn test_dir_get_previous() -> anyhow::Result<()> {
let store = MemoryStore::default();
let dir = Dir::new(store.clone());

// Initially, there's no previous version
assert!(dir.get_previous().is_none());

// Store the directory
let first_cid = dir.store().await?;

// Load the directory and create a new version
let mut loaded_dir = Dir::load(&first_cid, store.clone()).await?;
loaded_dir.put_entry("new_file", Cid::default().into())?;

// Store the new version
let second_cid = loaded_dir.store().await?;

// Load the new version
let new_version = Dir::load(&second_cid, store).await?;

// Now the previous and initial load CIDs are set
assert_eq!(new_version.get_previous(), Some(&first_cid));
assert_eq!(new_version.get_initial_load_cid(), Some(&second_cid));

Ok(())
}
}
2 changes: 1 addition & 1 deletion monofs/lib/filesystem/dir/ops.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ where
/// # }
/// ```
pub fn list(&self) -> FsResult<Vec<Utf8UnixPathSegment>> {
Ok(self.inner.entries.keys().cloned().collect())
Ok(self.get_entries().map(|(k, _)| k.clone()).collect())
}

/// Copies an entity from the source path to the target **directory**.
Expand Down
Loading

0 comments on commit b13fb99

Please sign in to comment.