From 83eee439166cad0c05cee569da6a417e47038f23 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Sun, 20 Oct 2024 17:48:38 +0100 Subject: [PATCH] feat: basic filesystem impl and other deps (#10) * feat: basic filesystem impl and other deps - implement content addressable basic fs - import IPLD store impl - create other subprojects like raft, ucan, did - add micro_vm provisioning benchmark - change paths fron UnixPathBuf to Utf8UnixPathBuf * fix: replace PathDirs in `find*` fns with a better mut crawling impl - this should be more efficient than PathDirs - and it solves the forking issue we had with PathDirs earlier - now our tests pass - emphasize the immutability nature of the filesystem * fix: clippy long doc pragaraph complaint * feat: implement more apis and fix cidlink `resolve_mut` issue - preventing stale Cids in `CidLink`s by making refactoring structure and fetching supporting async cid resolution. It means we will be able to store and get the latest cid when something changes. - implement file input/ouput stream * feat: implement softlink resolution - cid based softlink has problems and we may likely move to a path based impl in the future; also to maintain compat wih Unix - add more unit tests and doc tests * fix: silent softlink brokenlink error issue - also fix env_pair and rlimit doc tests * feat: add readme and make nightly clippy happy * fix: disable nightly checks cause clippy broken again --- .github/workflows/tests_and_checks.yml | 24 +- .gitignore | 2 +- Cargo.lock | 852 +++++++++++++++++- Cargo.toml | 51 +- README.md | 10 +- deny.toml | 26 +- monoagent/Cargo.toml | 13 + monoagent/lib/lib.rs | 8 + monobase/Cargo.toml | 6 +- monocore/Cargo.toml | 51 +- monocore/Makefile | 28 +- monocore/benches/microvm_provision.rs | 35 + monocore/examples/microvm_nop.rs | 43 + monocore/examples/microvm_shell.rs | 9 +- monocore/lib/config/monocore.rs | 2 +- monocore/lib/config/path_pair.rs | 20 +- monocore/lib/config/port_pair.rs | 4 +- monocore/lib/oci/distribution/docker.rs | 4 +- monocore/lib/runtime/vm/builder.rs | 27 +- monocore/lib/runtime/vm/env_pair.rs | 53 +- monocore/lib/runtime/vm/rlimit.rs | 24 +- monocore/lib/runtime/vm/vm.rs | 21 +- monofs/Cargo.toml | 20 +- monofs/README.md | 129 +++ monofs/examples/dir_ops.rs | 81 ++ monofs/examples/file_ops.rs | 57 ++ monofs/lib/config/default.rs | 6 + monofs/lib/config/mod.rs | 9 + monofs/lib/filesystem/dir/dir.rs | 668 ++++++++++++++ monofs/lib/filesystem/dir/find.rs | 385 ++++++++ monofs/lib/filesystem/dir/mod.rs | 14 + monofs/lib/filesystem/dir/ops.rs | 612 +++++++++++++ monofs/lib/filesystem/dir/segment.rs | 291 ++++++ monofs/lib/filesystem/entity.rs | 138 +++ monofs/lib/filesystem/eq.rs | 67 ++ monofs/lib/filesystem/error.rs | 166 ++++ monofs/lib/filesystem/file/file.rs | 428 +++++++++ monofs/lib/filesystem/file/io.rs | 180 ++++ monofs/lib/filesystem/file/mod.rs | 11 + monofs/lib/filesystem/kind.rs | 21 + .../lib/filesystem/link/cidlink/attributes.rs | 12 + monofs/lib/filesystem/link/cidlink/entity.rs | 102 +++ monofs/lib/filesystem/link/cidlink/mod.rs | 9 + monofs/lib/filesystem/link/link.rs | 195 ++++ monofs/lib/filesystem/link/mod.rs | 9 + monofs/lib/filesystem/metadata.rs | 153 ++++ monofs/lib/filesystem/mod.rs | 29 + monofs/lib/filesystem/resolvable.rs | 23 + monofs/lib/filesystem/softlink.rs | 563 ++++++++++++ monofs/lib/filesystem/storeswitch.rs | 14 + monofs/lib/lib.rs | 7 +- monofs/lib/store/membufferstore.rs | 117 +++ monofs/lib/store/mod.rs | 9 + monofs/lib/utils/mod.rs | 7 + monofs/lib/utils/path.rs | 122 +++ monoutils-did/Cargo.toml | 13 + monoutils-did/lib/lib.rs | 8 + monoutils-raft/Cargo.toml | 13 + monoutils-raft/lib/lib.rs | 8 + monoutils-store/Cargo.toml | 27 + monoutils-store/lib/chunker.rs | 23 + monoutils-store/lib/error.rs | 113 +++ .../lib/implementations/chunkers/constants.rs | 6 + .../lib/implementations/chunkers/fixed.rs | 110 +++ .../lib/implementations/chunkers/mod.rs | 11 + .../lib/implementations/chunkers/rabin.rs | 57 ++ .../lib/implementations/layouts/balanced.rs | 41 + .../lib/implementations/layouts/flat.rs | 547 +++++++++++ .../lib/implementations/layouts/mod.rs | 11 + .../lib/implementations/layouts/trickle.rs | 38 + monoutils-store/lib/implementations/mod.rs | 11 + .../lib/implementations/stores/dualstore.rs | 278 ++++++ .../lib/implementations/stores/memstore.rs | 314 +++++++ .../lib/implementations/stores/mod.rs | 11 + .../lib/implementations/stores/plcstore.rs | 80 ++ monoutils-store/lib/layout.rs | 45 + monoutils-store/lib/lib.rs | 41 + monoutils-store/lib/merkle.rs | 66 ++ monoutils-store/lib/references.rs | 97 ++ monoutils-store/lib/seekable.rs | 14 + monoutils-store/lib/storable.rs | 20 + monoutils-store/lib/store.rs | 197 ++++ monoutils-store/lib/utils.rs | 23 + monoutils-ucan/Cargo.toml | 13 + monoutils-ucan/lib/lib.rs | 8 + monovue/Cargo.toml | 6 +- 86 files changed, 8042 insertions(+), 175 deletions(-) create mode 100644 monoagent/Cargo.toml create mode 100644 monoagent/lib/lib.rs create mode 100644 monocore/benches/microvm_provision.rs create mode 100644 monocore/examples/microvm_nop.rs create mode 100644 monofs/README.md create mode 100644 monofs/examples/dir_ops.rs create mode 100644 monofs/examples/file_ops.rs create mode 100644 monofs/lib/config/default.rs create mode 100644 monofs/lib/config/mod.rs create mode 100644 monofs/lib/filesystem/dir/dir.rs create mode 100644 monofs/lib/filesystem/dir/find.rs create mode 100644 monofs/lib/filesystem/dir/mod.rs create mode 100644 monofs/lib/filesystem/dir/ops.rs create mode 100644 monofs/lib/filesystem/dir/segment.rs create mode 100644 monofs/lib/filesystem/entity.rs create mode 100644 monofs/lib/filesystem/eq.rs create mode 100644 monofs/lib/filesystem/error.rs create mode 100644 monofs/lib/filesystem/file/file.rs create mode 100644 monofs/lib/filesystem/file/io.rs create mode 100644 monofs/lib/filesystem/file/mod.rs create mode 100644 monofs/lib/filesystem/kind.rs create mode 100644 monofs/lib/filesystem/link/cidlink/attributes.rs create mode 100644 monofs/lib/filesystem/link/cidlink/entity.rs create mode 100644 monofs/lib/filesystem/link/cidlink/mod.rs create mode 100644 monofs/lib/filesystem/link/link.rs create mode 100644 monofs/lib/filesystem/link/mod.rs create mode 100644 monofs/lib/filesystem/metadata.rs create mode 100644 monofs/lib/filesystem/mod.rs create mode 100644 monofs/lib/filesystem/resolvable.rs create mode 100644 monofs/lib/filesystem/softlink.rs create mode 100644 monofs/lib/filesystem/storeswitch.rs create mode 100644 monofs/lib/store/membufferstore.rs create mode 100644 monofs/lib/store/mod.rs create mode 100644 monofs/lib/utils/mod.rs create mode 100644 monofs/lib/utils/path.rs create mode 100644 monoutils-did/Cargo.toml create mode 100644 monoutils-did/lib/lib.rs create mode 100644 monoutils-raft/Cargo.toml create mode 100644 monoutils-raft/lib/lib.rs create mode 100644 monoutils-store/Cargo.toml create mode 100644 monoutils-store/lib/chunker.rs create mode 100644 monoutils-store/lib/error.rs create mode 100644 monoutils-store/lib/implementations/chunkers/constants.rs create mode 100644 monoutils-store/lib/implementations/chunkers/fixed.rs create mode 100644 monoutils-store/lib/implementations/chunkers/mod.rs create mode 100644 monoutils-store/lib/implementations/chunkers/rabin.rs create mode 100644 monoutils-store/lib/implementations/layouts/balanced.rs create mode 100644 monoutils-store/lib/implementations/layouts/flat.rs create mode 100644 monoutils-store/lib/implementations/layouts/mod.rs create mode 100644 monoutils-store/lib/implementations/layouts/trickle.rs create mode 100644 monoutils-store/lib/implementations/mod.rs create mode 100644 monoutils-store/lib/implementations/stores/dualstore.rs create mode 100644 monoutils-store/lib/implementations/stores/memstore.rs create mode 100644 monoutils-store/lib/implementations/stores/mod.rs create mode 100644 monoutils-store/lib/implementations/stores/plcstore.rs create mode 100644 monoutils-store/lib/layout.rs create mode 100644 monoutils-store/lib/lib.rs create mode 100644 monoutils-store/lib/merkle.rs create mode 100644 monoutils-store/lib/references.rs create mode 100644 monoutils-store/lib/seekable.rs create mode 100644 monoutils-store/lib/storable.rs create mode 100644 monoutils-store/lib/store.rs create mode 100644 monoutils-store/lib/utils.rs create mode 100644 monoutils-ucan/Cargo.toml create mode 100644 monoutils-ucan/lib/lib.rs diff --git a/.github/workflows/tests_and_checks.yml b/.github/workflows/tests_and_checks.yml index fad276a..646307b 100644 --- a/.github/workflows/tests_and_checks.yml +++ b/.github/workflows/tests_and_checks.yml @@ -18,7 +18,7 @@ concurrency: cancel-in-progress: true jobs: - check-libkrun-changes: + check-libkrun-cache-changes: runs-on: ubuntu-latest outputs: should_build: ${{ steps.check_build.outputs.should_build }} @@ -53,13 +53,17 @@ jobs: elif [ $(git rev-list --count HEAD) -lt 2 ]; then echo "Only one commit in the repository, building libkrun" echo "should_build=true" >> $GITHUB_OUTPUT + elif git diff --name-only HEAD^ HEAD | grep -qE 'build_libkrun.sh|.github/workflows/tests_and_checks.yml'; then + echo "build_libkrun.sh or workflow file has changed, building libkrun" + echo "should_build=true" >> $GITHUB_OUTPUT else - git diff --name-only HEAD^ HEAD | grep -q 'build_libkrun.sh' && echo "should_build=true" >> $GITHUB_OUTPUT || echo "should_build=false" >> $GITHUB_OUTPUT + echo "No relevant changes, no need to build" + echo "should_build=false" >> $GITHUB_OUTPUT fi build-libkrun: - needs: check-libkrun-changes - if: needs.check-libkrun-changes.outputs.should_build == 'true' || github.event.inputs.force_build_libkrun == 'true' + needs: check-libkrun-cache-changes + if: needs.check-libkrun-cache-changes.outputs.should_build == 'true' || github.event.inputs.force_build_libkrun == 'true' runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -88,10 +92,10 @@ jobs: path: | build/libkrunfw/libkrunfw*.so* build/libkrun/target/release/libkrun*.so* - key: ${{ runner.os }}-libkrun-${{ needs.check-libkrun-changes.outputs.libkrun_hash }} + key: ${{ runner.os }}-libkrun-${{ needs.check-libkrun-cache-changes.outputs.libkrun_hash }} run-checks: - needs: [check-libkrun-changes, build-libkrun] + needs: [check-libkrun-cache-changes, build-libkrun] if: always() runs-on: ubuntu-latest strategy: @@ -99,7 +103,7 @@ jobs: matrix: rust-toolchain: - stable - - nightly + # - nightly # Nightly Clippy having concussion again! steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -113,7 +117,7 @@ jobs: path: | build/libkrunfw/libkrunfw*.so* build/libkrun/target/release/libkrun*.so* - key: ${{ runner.os }}-libkrun-${{ needs.check-libkrun-changes.outputs.libkrun_hash }} + key: ${{ runner.os }}-libkrun-${{ needs.check-libkrun-cache-changes.outputs.libkrun_hash }} - name: Install Rust Toolchain uses: actions-rs/toolchain@v1 @@ -166,7 +170,7 @@ jobs: cargo build --release run-tests: - needs: [check-libkrun-changes, build-libkrun] + needs: [check-libkrun-cache-changes, build-libkrun] if: always() runs-on: ubuntu-latest strategy: @@ -193,7 +197,7 @@ jobs: path: | build/libkrunfw/libkrunfw*.so* build/libkrun/target/release/libkrun*.so* - key: ${{ runner.os }}-libkrun-${{ needs.check-libkrun-changes.outputs.libkrun_hash }} + key: ${{ runner.os }}-libkrun-${{ needs.check-libkrun-cache-changes.outputs.libkrun_hash }} - name: Print Cache run: tree -L 2 build/ diff --git a/.gitignore b/.gitignore index 9a49173..922c421 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # Generated by Cargo # will have compiled files and executables -/target/ +**/target/ # These are backup files generated by rustfmt **/*.rs.bk diff --git a/Cargo.lock b/Cargo.lock index 165e3d8..f5502c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -41,6 +53,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.15" @@ -96,6 +114,57 @@ version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "async-once-cell" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288f83726785267c6f2ef073a3d83dc3f9b81464e9f99898240cced85fce35a" + +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "async-trait" version = "0.1.83" @@ -104,7 +173,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] @@ -189,6 +258,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + [[package]] name = "base64" version = "0.22.1" @@ -207,6 +282,41 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "blake2b_simd" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23285ad32269793932e830392f2fe2f83e26488fd3ec778883a93c8323735780" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "blake2s_simd" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94230421e395b9920d23df13ea5d77a20e1725331f90fbbf6df6040b33f756ae" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "blake3" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -234,6 +344,21 @@ version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + [[package]] name = "cc" version = "1.1.30" @@ -264,6 +389,61 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd94671561e36e4e7de75f753f577edafb0e7c05d6e4547229fdf7938fbcd2c3" +dependencies = [ + "core2", + "multibase", + "multihash 0.18.1", + "serde", + "serde_bytes", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "cid" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3147d8272e8fa0ccd29ce51194dd98f79ddfb8191ba9e3409884e751798acf3a" +dependencies = [ + "core2", + "multibase", + "multihash 0.19.1", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + [[package]] name = "clap" version = "4.5.20" @@ -295,7 +475,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] @@ -310,6 +490,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "core-foundation" version = "0.9.4" @@ -326,6 +512,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + [[package]] name = "cpufeatures" version = "0.2.14" @@ -335,6 +530,73 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -366,7 +628,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.79", ] [[package]] @@ -377,7 +639,33 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.79", +] + +[[package]] +name = "data-encoding" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" + +[[package]] +name = "data-encoding-macro" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1559b6cba622276d6d63706db152618eeb15b89b3e4041446b05876e352e639" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332d754c0af53bc87c108fed664d121ecf59207ec4196041f04d6ab9002ad33f" +dependencies = [ + "data-encoding", + "syn 1.0.109", ] [[package]] @@ -398,7 +686,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] @@ -408,7 +696,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn", + "syn 2.0.79", ] [[package]] @@ -448,6 +736,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "encoding_rs" version = "0.8.34" @@ -506,6 +800,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -586,7 +886,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] @@ -651,7 +951,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] @@ -679,11 +979,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heck" @@ -703,6 +1018,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -889,18 +1210,49 @@ dependencies = [ "web-sys", ] +[[package]] +name = "ipld-core" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ede82a79e134f179f4b29b5fdb1eb92bd1b38c4dfea394c539051150a21b9b" +dependencies = [ + "cid 0.11.1", + "serde", + "serde_bytes", +] + [[package]] name = "ipnet" version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi 0.4.0", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -916,6 +1268,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -924,9 +1285,99 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.161" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" + +[[package]] +name = "libipld" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1ccd6b8ffb3afee7081fcaec00e1b099fd1c7ccf35ba5729d88538fcc3b4599" +dependencies = [ + "fnv", + "libipld-cbor", + "libipld-cbor-derive", + "libipld-core", + "libipld-json", + "libipld-macro", + "libipld-pb", + "log", + "multihash 0.18.1", + "thiserror", +] + +[[package]] +name = "libipld-cbor" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77d98c9d1747aa5eef1cf099cd648c3fd2d235249f5fed07522aaebc348e423b" +dependencies = [ + "byteorder", + "libipld-core", + "thiserror", +] + +[[package]] +name = "libipld-cbor-derive" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5ba3a729b72973e456a1812b0afe2e176a376c1836cc1528e9fc98ae8cb838" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure", +] + +[[package]] +name = "libipld-core" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "5acd707e8d8b092e967b2af978ed84709eaded82b75effe6cb6f6cc797ef8158" +dependencies = [ + "anyhow", + "cid 0.10.1", + "core2", + "multibase", + "multihash 0.18.1", + "serde", + "thiserror", +] + +[[package]] +name = "libipld-json" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25856def940047b07b25c33d4e66d248597049ab0202085215dc4dca0487731c" +dependencies = [ + "libipld-core", + "multihash 0.18.1", + "serde", + "serde_json", +] + +[[package]] +name = "libipld-macro" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71171c54214f866ae6722f3027f81dff0931e600e5a61e6b1b6a49ca0b5ed4ae" +dependencies = [ + "libipld-core", +] + +[[package]] +name = "libipld-pb" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f2d0f866c4cd5dc9aa8068c429ba478d2882a3a4b70ab56f7e9a0eddf5d16f" +dependencies = [ + "bytes", + "libipld-core", + "quick-protobuf", + "thiserror", +] [[package]] name = "libredox" @@ -960,6 +1411,15 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown", +] + [[package]] name = "matchers" version = "0.1.0" @@ -1002,12 +1462,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "wasi", "windows-sys 0.52.0", ] +[[package]] +name = "monoagent" +version = "0.1.0" + [[package]] name = "monobase" version = "0.1.0" @@ -1022,12 +1486,14 @@ dependencies = [ "bytes", "chrono", "clap", + "criterion", "dirs", "dotenvy", "futures", "getset", "hex", "lazy_static", + "libc", "oci-spec", "reqwest", "reqwest-middleware", @@ -1039,7 +1505,7 @@ dependencies = [ "test-log", "thiserror", "tokio", - "toml", + "toml 0.8.19", "tracing", "tracing-subscriber", "typed-builder", @@ -1050,11 +1516,113 @@ dependencies = [ [[package]] name = "monofs" version = "0.1.0" +dependencies = [ + "aliasable", + "anyhow", + "async-once-cell", + "async-recursion", + "bytes", + "chrono", + "futures", + "getset", + "monoutils-store", + "serde", + "serde_json", + "thiserror", + "tokio", + "typed-path", +] + +[[package]] +name = "monoutils-did" +version = "0.1.0" + +[[package]] +name = "monoutils-raft" +version = "0.1.0" + +[[package]] +name = "monoutils-store" +version = "0.1.0" +dependencies = [ + "aliasable", + "anyhow", + "async-stream", + "bytes", + "futures", + "hex", + "libipld", + "lru", + "serde", + "serde_ipld_dagcbor", + "thiserror", + "tokio", + "tokio-util", +] + +[[package]] +name = "monoutils-ucan" +version = "0.1.0" [[package]] name = "monovue" version = "0.1.0" +[[package]] +name = "multibase" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +dependencies = [ + "base-x", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd8a792c1694c6da4f68db0a9d707c72bd260994da179e6030a5dcee00bb815" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "blake3", + "core2", + "digest", + "multihash-derive", + "serde", + "serde-big-array", + "sha2", + "sha3", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "multihash" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076d548d76a0e2a0d4ab471d0b1c36c577786dfc4471242035d97a12a735c492" +dependencies = [ + "core2", + "serde", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "multihash-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6d4752e6230d8ef7adf7bd5d8c4b1f6561c1014c5ba9a37445ccefe18aa1db" +dependencies = [ + "proc-macro-crate", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure", +] + [[package]] name = "native-tls" version = "0.2.12" @@ -1103,8 +1671,7 @@ dependencies = [ [[package]] name = "oci-spec" version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cee185ce7cf1cce45e194e34cd87c0bad7ff0aa2e8917009a2da4f7b31fb363" +source = "git+https://github.com/containers/oci-spec-rs?branch=main#e436a3024778cf734336952f4d37b69864488d0e" dependencies = [ "derive_builder", "getset", @@ -1122,6 +1689,12 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "oorandom" +version = "11.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" + [[package]] name = "openssl" version = "0.10.66" @@ -1145,7 +1718,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] @@ -1250,6 +1823,34 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -1259,6 +1860,40 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" +dependencies = [ + "thiserror", + "toml 0.5.11", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro-error-attr2" version = "2.0.0" @@ -1278,7 +1913,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] @@ -1290,6 +1925,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-protobuf" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f" +dependencies = [ + "byteorder", +] + [[package]] name = "quote" version = "1.0.37" @@ -1329,6 +1973,26 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -1360,9 +2024,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", @@ -1577,6 +2241,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.26" @@ -1624,6 +2297,24 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-big-array" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd31f59f6fe2b0c055371bb2f16d7f0aa7d8881676c04a55b1596d1a17cd10a4" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.210" @@ -1632,14 +2323,26 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.79", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded35fbe4ab8fdec1f1d14b4daff2206b1eada4d6e708cb451d464d2d965f493" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", ] [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -1689,6 +2392,16 @@ dependencies = [ "digest", ] +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1778,7 +2491,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.79", ] [[package]] @@ -1787,6 +2500,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.79" @@ -1813,6 +2537,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + [[package]] name = "system-configuration" version = "0.6.1" @@ -1866,7 +2602,7 @@ checksum = "5999e24eaa32083191ba4e425deb75cdf25efefabe5aaccb7446dd0d4122a3f5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] @@ -1886,7 +2622,7 @@ checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] @@ -1899,6 +2635,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -1940,7 +2686,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] @@ -1977,6 +2723,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + [[package]] name = "toml" version = "0.8.19" @@ -2059,7 +2814,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] @@ -2124,7 +2879,7 @@ checksum = "560b82d656506509d43abe30e0ba64c56b1953ab3d4fe7ba5902747a7a3cedd5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] @@ -2160,6 +2915,24 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + [[package]] name = "untrusted" version = "0.9.0" @@ -2211,6 +2984,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2248,7 +3031,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.79", "wasm-bindgen-shared", ] @@ -2282,7 +3065,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.79", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2347,6 +3130,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -2578,7 +3370,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.79", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index be05a02..04ae6d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,19 +1,60 @@ [workspace] -members = ["monobase", "monocore", "monofs", "monovue"] +members = [ + "monoagent", + "monobase", + "monocore", + "monofs", + "monoutils-did", + "monoutils-raft", + "monoutils-store", + "monoutils-ucan", + "monovue", +] resolver = "2" [workspace.package] +authors = ["Stephen Akinyemi "] +repository = "https://github.com/appcypher/monocore" +version = "0.1.0" license = "Apache-2.0" +edition = "2021" [workspace.dependencies] +async-stream = "0.3.5" +async-trait = "0.1" +dirs = "5.0" +hex = "0.4" +libc = "0.2" +axum = "0.7.7" +bytes = "1.6.0" +libipld = "0.16.0" serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +structstruck = "0.4" +xattr = "1.3" +sha2 = "0.10" thiserror = "1.0" anyhow = "1.0" futures = "0.3" tokio = { version = "1.34", features = ["full"] } +tokio-util = { version = "0.7.11", features = ["io"] } dotenvy = "0.15" -reqwest = { version = "0.12", features = ["json"] } -tracing = "0.1.40" +tracing = "0.1" tracing-subscriber = "0.3" -clap = { version = "4.5.16", features = ["color", "derive"] } -getset = "0.1.3" +clap = { version = "4.5", features = ["color", "derive"] } +getset = "0.1" +lazy_static = "1.5" +regex = "1.11" +reqwest = { version = "0.12", features = ["stream", "json"] } +reqwest-middleware = "0.3" +reqwest-retry = "0.6" +monoutils-ucan = { path = "./monoutils-ucan" } +monoutils-did = { path = "./monoutils-did" } +monoutils-store = { path = "./monoutils-store" } +monoutils-raft = { path = "./monoutils-raft" } +chrono = "0.4" +criterion = "0.5" +test-log = "0.2" +typed-path = "0.9" +toml = "0.8" +typed-builder = "0.20" diff --git a/README.md b/README.md index 0195fe1..561ea3a 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Follow these steps to set up monocore for development: ### Prerequisites - [Git][git_home] -- [Rust toolchain (latest stable version)][rustup_home] +- [Rust toolchain][rustup_home] (latest stable version) - On macOS: [Homebrew][brew_home] ### Setup @@ -89,8 +89,14 @@ Follow these steps to set up monocore for development: make example microvm_shell ``` -## Contributing +5. **Run benchmarks** + + ```sh + cd monocore # Ensure you are in the monocore subdirectory + make bench microvm_provision + ``` +## Contributing 1. **Read the [CONTRIBUTING.md](./CONTRIBUTING.md) file** diff --git a/deny.toml b/deny.toml index d34c746..d9c7f59 100644 --- a/deny.toml +++ b/deny.toml @@ -50,9 +50,9 @@ db-urls = ["https://github.com/rustsec/advisory-db"] ignore = [ "RUSTSEC-2021-0145", # atty on windows only "RUSTSEC-2023-0071", # Impacts rsa crate, which is only used in dev, see - # https://github.com/RustCrypto/RSA/pull/394 for remediation + # https://github.com/RustCrypto/RSA/pull/394 for remediation "RUSTSEC-2024-0336", # Ignore a DOS issue w/ rustls-0.20.9. This will go - # away when we update opentelemetry-otlp soon. + # away when we update opentelemetry-otlp soon. { id = "RUSTSEC-2020-0168", reason = "Not planning to force upgrade to mach2 yet" }, { id = "RUSTSEC-2024-0320", reason = "Not planning to force upgrade to rust-yaml2 yet" }, ] @@ -83,13 +83,13 @@ allow = [ "BSD-3-Clause", "MPL-2.0", "ISC", - "Zlib" + "Zlib", ] # List of explicitly disallowed licenses # See https://spdx.org/licenses/ for list of possible licenses # [possible values: any SPDX 3.7 short identifier (+ optional exception)]. # deny = [ - #"Nokia", +#"Nokia", # ] # Lint level for licenses considered copyleft # copyleft = "deny" @@ -117,9 +117,15 @@ exceptions = [ # use data from the unicode tables to generate the tables which are # included in the application. We do not distribute those data files so # this is not a problem for us. See https://github.com/dtolnay/unicode-ident/pull/9/files - { allow = ["Unicode-DFS-2016"], name = "unicode-ident", version = "*"}, - { allow = ["OpenSSL"], name = "ring", version = "*" }, - { allow = ["MPL-2.0"], name = "webpki-roots", version = "*"}, + { allow = [ + "Unicode-DFS-2016", + ], name = "unicode-ident", version = "*" }, + { allow = [ + "OpenSSL", + ], name = "ring", version = "*" }, + { allow = [ + "MPL-2.0", + ], name = "webpki-roots", version = "*" }, ] # Some crates don't have (easily) machine readable licensing information, @@ -128,9 +134,7 @@ exceptions = [ [[licenses.clarify]] name = "ring" expression = "MIT AND ISC AND OpenSSL" -license-files = [ - { path = "LICENSE", hash = 0xbd0eed23 } -] +license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }] [licenses.private] # If true, ignores workspace crates that aren't published, or are only @@ -189,7 +193,7 @@ unknown-git = "deny" # if not specified. If it is specified but empty, no registries are allowed. allow-registry = ["https://github.com/rust-lang/crates.io-index"] # List of URLs for allowed Git repositories -allow-git = [] +allow-git = ["https://github.com/containers/oci-spec-rs"] #[sources.allow-org] # 1 or more github.com organizations to allow git sources for diff --git a/monoagent/Cargo.toml b/monoagent/Cargo.toml new file mode 100644 index 0000000..92b699e --- /dev/null +++ b/monoagent/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "monoagent" +version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "monoagent" +path = "lib/lib.rs" + +[dependencies] diff --git a/monoagent/lib/lib.rs b/monoagent/lib/lib.rs new file mode 100644 index 0000000..b8f4f6a --- /dev/null +++ b/monoagent/lib/lib.rs @@ -0,0 +1,8 @@ +//! `monovue` is a UI for working with monocore. + +#![warn(missing_docs)] +#![allow(clippy::module_inception)] + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- diff --git a/monobase/Cargo.toml b/monobase/Cargo.toml index 0eb456d..927fddb 100644 --- a/monobase/Cargo.toml +++ b/monobase/Cargo.toml @@ -1,8 +1,10 @@ [package] name = "monobase" -version = "0.1.0" -edition = "2021" +version.workspace = true license.workspace = true +repository.workspace = true +authors.workspace = true +edition.workspace = true [lib] name = "monobase" diff --git a/monocore/Cargo.toml b/monocore/Cargo.toml index 84a6379..75f2a5b 100644 --- a/monocore/Cargo.toml +++ b/monocore/Cargo.toml @@ -1,8 +1,10 @@ [package] name = "monocore" -version = "0.1.0" -edition = "2021" +version.workspace = true license.workspace = true +repository.workspace = true +authors.workspace = true +edition.workspace = true [lib] name = "monocore" @@ -12,35 +14,42 @@ path = "lib/lib.rs" name = "monocore" path = "bin/monocore.rs" +[[bench]] +name = "microvm_provision" +harness = false + [dependencies] anyhow.workspace = true -async-trait = "0.1.83" -axum = "0.7.7" -bytes = "1.7.2" -chrono = { version = "0.4.38", features = ["serde"] } +async-trait.workspace = true +axum.workspace = true +bytes.workspace = true +chrono.workspace = true clap.workspace = true -dirs = "5.0.1" +criterion.workspace = true +dirs.workspace = true dotenvy.workspace = true futures.workspace = true getset.workspace = true -hex = "0.4.3" -lazy_static = "1.5.0" -oci-spec = "0.7.0" -reqwest = { workspace = true, features = ["stream"] } -reqwest-middleware = "0.3.3" -reqwest-retry = "0.6.1" +hex.workspace = true +lazy_static.workspace = true +libc.workspace = true +oci-spec = { git = "https://github.com/containers/oci-spec-rs", branch = "main" } +reqwest.workspace = true +reqwest-middleware.workspace = true +reqwest-retry.workspace = true serde.workspace = true -serde_json = "1.0.128" -sha2 = "0.10.8" -structstruck = "0.4.1" +serde_json.workspace = true +sha2.workspace = true +structstruck.workspace = true thiserror.workspace = true tokio.workspace = true -toml = "0.8.19" +toml.workspace = true tracing.workspace = true tracing-subscriber.workspace = true -typed-builder = "0.20.0" -typed-path = "0.9.2" -xattr = "1.3.1" +typed-builder.workspace = true +typed-path.workspace = true +xattr.workspace = true [dev-dependencies] -test-log = "0.2.16" +test-log.workspace = true +criterion.workspace = true diff --git a/monocore/Makefile b/monocore/Makefile index 87cb23f..91371d0 100644 --- a/monocore/Makefile +++ b/monocore/Makefile @@ -9,31 +9,32 @@ ifeq ($(ARCH),x86_64) endif PREFIX ?= /usr/local -RELEASE_BIN := ../target/release/monocore -EXAMPLES_DIR := ../target/debug/examples +MONOCORE_RELEASE_BIN := ../target/release/monocore +EXAMPLES_DIR := ../target/release/examples FIXTURES_DIR := fixtures DISTRO_ROOTFS := rootfs-alpine BUILD_DIR := build +BENCHES_DIR := ../target/release # Library paths DARWIN_LIB_PATH := /usr/local/lib LINUX_LIB_PATH := /usr/local/lib64 # Phony targets -.PHONY: all install clean example unpack_rootfs +.PHONY: all install clean example unpack_rootfs bench # Default target -all: $(RELEASE_BIN) +all: $(MONOCORE_RELEASE_BIN) # Build the release binary -$(RELEASE_BIN): +$(MONOCORE_RELEASE_BIN): cargo build --release -p monocore ifeq ($(OS),Darwin) codesign --entitlements monocore.entitlements --force -s - $@ endif # Install the binary -install: $(RELEASE_BIN) +install: $(MONOCORE_RELEASE_BIN) install -d $(DESTDIR)$(PREFIX)/bin install -m 755 $< $(DESTDIR)$(PREFIX)/bin @@ -60,13 +61,24 @@ example: unpack_rootfs _run_example: ifeq ($(OS),Darwin) - cargo build --example $(EXAMPLE_NAME) + cargo build --example $(EXAMPLE_NAME) --release codesign --entitlements monocore.entitlements --force -s - $(EXAMPLES_DIR)/$(EXAMPLE_NAME) DYLD_LIBRARY_PATH=$(DARWIN_LIB_PATH):$$DYLD_LIBRARY_PATH $(EXAMPLES_DIR)/$(EXAMPLE_NAME) else - LD_LIBRARY_PATH=$(LINUX_LIB_PATH):$$LD_LIBRARY_PATH cargo run --example $(EXAMPLE_NAME) + LD_LIBRARY_PATH=$(LINUX_LIB_PATH):$$LD_LIBRARY_PATH cargo run --example $(EXAMPLE_NAME) --release endif +# Run benchmarks +bench: + @if [ -z "$(word 2,$(MAKECMDGOALS))" ]; then \ + echo "Usage: make bench "; \ + exit 1; \ + fi + @$(MAKE) _run_bench BENCH_NAME=$(word 2,$(MAKECMDGOALS)) + +_run_bench: + cargo bench --bench $(BENCH_NAME) + # Catch-all target to allow example names %: @: diff --git a/monocore/benches/microvm_provision.rs b/monocore/benches/microvm_provision.rs new file mode 100644 index 0000000..00439b9 --- /dev/null +++ b/monocore/benches/microvm_provision.rs @@ -0,0 +1,35 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use std::{process::Command, time::Duration}; + +//-------------------------------------------------------------------------------------------------- +// Benchmark +//-------------------------------------------------------------------------------------------------- + +fn benchmark_microvm_nop(c: &mut Criterion) { + // First, execute the make command + let make_status = Command::new("make") + .args(&["example", "microvm_nop"]) + .status() + .expect("Failed to execute make command"); + + if !make_status.success() { + panic!("make example microvm_nop failed"); + } + + // Now benchmark the microvm_nop example + c.bench_function("microvm_nop", |b| { + b.iter(|| { + let output = Command::new("../target/release/examples/microvm_nop") + .output() + .expect("Failed to execute microvm_nop"); + assert!(output.status.success()); + }) + }); +} + +criterion_group! { + name = benches; + config = Criterion::default().sample_size(10).measurement_time(Duration::from_secs(10)); + targets = benchmark_microvm_nop +} +criterion_main!(benches); diff --git a/monocore/examples/microvm_nop.rs b/monocore/examples/microvm_nop.rs new file mode 100644 index 0000000..82f2c3a --- /dev/null +++ b/monocore/examples/microvm_nop.rs @@ -0,0 +1,43 @@ +//! If you are trying to run this example, please make sure to run `make example microvm_nop` from +//! the `monocore` subdirectory + +use monocore::runtime::MicroVM; + +//-------------------------------------------------------------------------------------------------- +// Function: main +//-------------------------------------------------------------------------------------------------- + +fn main() -> anyhow::Result<()> { + // Get the current architecture + let arch = get_current_arch(); + + // Use the architecture-specific build directory + let rootfs_path = format!("build/rootfs-alpine-{}", arch); + + // Build the microVM + let vm = MicroVM::builder() + .root_path(&rootfs_path) + .exec_path("/bin/true") + .ram_mib(1024) + .build()?; + + // Start the microVM + vm.start(); + + Ok(()) +} + +//-------------------------------------------------------------------------------------------------- +// Function: * +//-------------------------------------------------------------------------------------------------- + +// Add this function to determine the current architecture +fn get_current_arch() -> &'static str { + if cfg!(target_arch = "x86_64") { + "x86_64" + } else if cfg!(target_arch = "aarch64") { + "arm64" + } else { + panic!("Unsupported architecture") + } +} diff --git a/monocore/examples/microvm_shell.rs b/monocore/examples/microvm_shell.rs index b0edec5..d3e1f6d 100644 --- a/monocore/examples/microvm_shell.rs +++ b/monocore/examples/microvm_shell.rs @@ -1,7 +1,10 @@ +//! If you are trying to run this example, please make sure to run `make example microvm_shell` from +//! the `monocore` subdirectory + use monocore::runtime::MicroVM; //-------------------------------------------------------------------------------------------------- -// Main +// Function: main //-------------------------------------------------------------------------------------------------- fn main() -> anyhow::Result<()> { @@ -30,6 +33,10 @@ fn main() -> anyhow::Result<()> { Ok(()) } +//-------------------------------------------------------------------------------------------------- +// Function: * +//-------------------------------------------------------------------------------------------------- + // Set an extended attribute on a file fn set_xattr(path: impl AsRef, name: &str, value: &[u8]) -> anyhow::Result<()> { xattr::set(path, name, value)?; diff --git a/monocore/lib/config/monocore.rs b/monocore/lib/config/monocore.rs index 25cd7f1..61ce8ee 100644 --- a/monocore/lib/config/monocore.rs +++ b/monocore/lib/config/monocore.rs @@ -18,7 +18,7 @@ use super::{PathPair, PortPair}; strike! { /// The monocore configuration. #[strikethrough[derive(Debug, Deserialize, Serialize, TypedBuilder, PartialEq, Getters, Setters)]] - #[getset(get_mut = "pub", get = "pub", set = "pub")] + #[getset(get_mut = "pub", get = "pub with_prefix", set = "pub with_prefix")] pub struct Monocore { /// The services to run. #[serde(rename = "service")] diff --git a/monocore/lib/config/path_pair.rs b/monocore/lib/config/path_pair.rs index 92c3a7b..2c8c4d5 100644 --- a/monocore/lib/config/path_pair.rs +++ b/monocore/lib/config/path_pair.rs @@ -1,7 +1,7 @@ use std::{fmt, str::FromStr}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use typed_path::UnixPathBuf; +use typed_path::Utf8UnixPathBuf; use crate::MonocoreError; @@ -15,13 +15,13 @@ pub enum PathPair { /// The guest path and host path are distinct. Distinct { /// The guest path. - guest: UnixPathBuf, + guest: Utf8UnixPathBuf, /// The host path. - host: UnixPathBuf, + host: Utf8UnixPathBuf, }, /// The guest path and host path are the same. - Same(UnixPathBuf), + Same(Utf8UnixPathBuf), } //-------------------------------------------------------------------------------------------------- @@ -30,24 +30,24 @@ pub enum PathPair { impl PathPair { /// Creates a new `PathPair` with the same host and guest path. - pub fn with_same(path: UnixPathBuf) -> Self { + pub fn with_same(path: Utf8UnixPathBuf) -> Self { Self::Same(path) } /// Creates a new `PathPair` with distinct guest and host paths. - pub fn with_distinct(guest: UnixPathBuf, host: UnixPathBuf) -> Self { + pub fn with_distinct(guest: Utf8UnixPathBuf, host: Utf8UnixPathBuf) -> Self { Self::Distinct { guest, host } } /// Returns the host path. - pub fn host(&self) -> &UnixPathBuf { + pub fn get_host(&self) -> &Utf8UnixPathBuf { match self { Self::Distinct { host, .. } | Self::Same(host) => host, } } /// Returns the guest path. - pub fn guest(&self) -> &UnixPathBuf { + pub fn get_guest(&self) -> &Utf8UnixPathBuf { match self { Self::Distinct { guest, .. } | Self::Same(guest) => guest, } @@ -91,9 +91,9 @@ impl fmt::Display for PathPair { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Distinct { guest, host } => { - write!(f, "{}:{}", guest.to_string_lossy(), host.to_string_lossy()) + write!(f, "{}:{}", guest, host) } - Self::Same(path) => write!(f, "{}:{}", path.to_string_lossy(), path.to_string_lossy()), + Self::Same(path) => write!(f, "{}:{}", path, path), } } } diff --git a/monocore/lib/config/port_pair.rs b/monocore/lib/config/port_pair.rs index 468a0ac..8ace3e6 100644 --- a/monocore/lib/config/port_pair.rs +++ b/monocore/lib/config/port_pair.rs @@ -40,14 +40,14 @@ impl PortPair { } /// Returns the host port. - pub fn host(&self) -> u16 { + pub fn get_host(&self) -> u16 { match self { Self::Distinct { host, .. } | Self::Same(host) => *host, } } /// Returns the guest port. - pub fn guest(&self) -> u16 { + pub fn get_guest(&self) -> u16 { match self { Self::Distinct { guest, .. } | Self::Same(guest) => *guest, } diff --git a/monocore/lib/oci/distribution/docker.rs b/monocore/lib/oci/distribution/docker.rs index a6436a0..18c6a4f 100644 --- a/monocore/lib/oci/distribution/docker.rs +++ b/monocore/lib/oci/distribution/docker.rs @@ -65,7 +65,7 @@ const DOCKER_CONFIG_MIME_TYPE: &str = "application/vnd.docker.container.image.v1 /// [OCI Distribution Spec]: https://distribution.github.io/distribution/spec/manifest-v2-2/#image-manifest-version-2-schema-2 /// [Docker Registry API]: https://distribution.github.io/distribution/spec/api/#introduction #[derive(Debug, Getters, Setters)] -#[getset(get = "pub", set = "pub")] +#[getset(get = "pub with_prefix", set = "pub with_prefix")] pub struct DockerRegistry { /// The HTTP client used to make requests to the Docker registry. client: ClientWithMiddleware, @@ -76,7 +76,7 @@ pub struct DockerRegistry { /// Stores authentication credentials obtained from the Docker registry, including tokens and expiration details. #[derive(Debug, Serialize, Deserialize, Getters, Setters)] -#[getset(get = "pub", set = "pub")] +#[getset(get = "pub with_prefix", set = "pub with_prefix")] pub struct DockerAuthMaterial { /// The token used to authenticate requests to the Docker registry. token: String, diff --git a/monocore/lib/runtime/vm/builder.rs b/monocore/lib/runtime/vm/builder.rs index 3c4bb4f..aa3c825 100644 --- a/monocore/lib/runtime/vm/builder.rs +++ b/monocore/lib/runtime/vm/builder.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use typed_path::UnixPathBuf; +use typed_path::Utf8UnixPathBuf; use crate::{ config::{PathPair, PortPair, DEFAULT_NUM_VCPUS}, @@ -20,7 +20,7 @@ use super::{EnvPair, LinuxRlimit, LogLevel, MicroVM, MicroVMConfig}; /// RAM size, virtio-fs mounts, port mappings, resource limits, working directory, executable path, /// arguments, environment variables, and console output. /// -/// # Examples +/// ## Examples /// /// ```rust /// use monocore::runtime::{MicroVMBuilder, LogLevel}; @@ -52,11 +52,11 @@ pub struct MicroVMBuilder { virtiofs: Vec, port_map: Vec, rlimits: Vec, - workdir_path: Option, - exec_path: Option, + workdir_path: Option, + exec_path: Option, argv: Vec, env: Vec, - console_output: Option, + console_output: Option, } //-------------------------------------------------------------------------------------------------- @@ -204,7 +204,7 @@ impl MicroVMBuilder { /// /// MicroVMBuilder::default().workdir_path("/path/to/workdir"); /// ``` - pub fn workdir_path(mut self, workdir_path: impl Into) -> Self { + pub fn workdir_path(mut self, workdir_path: impl Into) -> Self { self.workdir_path = Some(workdir_path.into()); self } @@ -218,7 +218,7 @@ impl MicroVMBuilder { /// /// MicroVMBuilder::default().exec_path("/path/to/exec"); /// ``` - pub fn exec_path(mut self, exec_path: impl Into) -> Self { + pub fn exec_path(mut self, exec_path: impl Into) -> Self { self.exec_path = Some(exec_path.into()); self } @@ -263,7 +263,7 @@ impl MicroVMBuilder { /// /// MicroVMBuilder::default().console_output("/tmp/console.log"); /// ``` - pub fn console_output(mut self, console_output: impl Into) -> Self { + pub fn console_output(mut self, console_output: impl Into) -> Self { self.console_output = Some(console_output.into()); self } @@ -278,7 +278,7 @@ impl MicroVMBuilder { /// /// Returns a `Result` containing the built `MicroVM` instance if successful, or a `MonocoreError` if there was an error. /// - /// # Examples + /// ## Examples /// /// ```rust /// # use monocore::runtime::MicroVMBuilder; @@ -366,8 +366,11 @@ mod tests { assert_eq!(builder.virtiofs, ["/guest/mount:/host/mount".parse()?]); assert_eq!(builder.port_map, ["8080:80".parse()?]); assert_eq!(builder.rlimits, ["RLIMIT_NOFILE=1024:1024".parse()?]); - assert_eq!(builder.workdir_path, Some(UnixPathBuf::from(workdir_path))); - assert_eq!(builder.exec_path, Some(UnixPathBuf::from(exec_path))); + assert_eq!( + builder.workdir_path, + Some(Utf8UnixPathBuf::from(workdir_path)) + ); + assert_eq!(builder.exec_path, Some(Utf8UnixPathBuf::from(exec_path))); assert_eq!(builder.argv, ["arg1".to_string(), "arg2".to_string()]); assert_eq!( builder.env, @@ -375,7 +378,7 @@ mod tests { ); assert_eq!( builder.console_output, - Some(UnixPathBuf::from("/tmp/console.log")) + Some(Utf8UnixPathBuf::from("/tmp/console.log")) ); Ok(()) } diff --git a/monocore/lib/runtime/vm/env_pair.rs b/monocore/lib/runtime/vm/env_pair.rs index 2cba0b0..6a99eae 100644 --- a/monocore/lib/runtime/vm/env_pair.rs +++ b/monocore/lib/runtime/vm/env_pair.rs @@ -1,7 +1,7 @@ use crate::MonocoreError; use getset::Getters; use serde::{Deserialize, Serialize}; -use std::{ffi::OsString, fmt, str::FromStr}; +use std::{fmt, str::FromStr}; //-------------------------------------------------------------------------------------------------- // Types @@ -12,7 +12,7 @@ use std::{ffi::OsString, fmt, str::FromStr}; /// This struct encapsulates a variable name and its corresponding value. /// It is used to manage environment variables for processes. /// -/// # Examples +/// ## Examples /// /// ``` /// use monocore::runtime::EnvPair; @@ -21,23 +21,23 @@ use std::{ffi::OsString, fmt, str::FromStr}; /// // Create a new environment variable pair /// let env_pair = EnvPair::new("PATH", "/usr/local/bin:/usr/bin"); /// -/// assert_eq!(env_pair.var().to_str().unwrap(), "PATH"); -/// assert_eq!(env_pair.value().to_str().unwrap(), "/usr/local/bin:/usr/bin"); +/// assert_eq!(env_pair.get_var(), "PATH"); +/// assert_eq!(env_pair.get_value(), "/usr/local/bin:/usr/bin"); /// /// // Parse an environment variable pair from a string /// let env_pair = EnvPair::from_str("USER=alice").unwrap(); /// -/// assert_eq!(env_pair.var().to_str().unwrap(), "USER"); -/// assert_eq!(env_pair.value().to_str().unwrap(), "alice"); +/// assert_eq!(env_pair.get_var(), "USER"); +/// assert_eq!(env_pair.get_value(), "alice"); /// ``` #[derive(Debug, Clone, PartialEq, Eq, Getters)] -#[getset(get = "pub")] +#[getset(get = "pub with_prefix")] pub struct EnvPair { /// The environment variable name. - var: OsString, + var: String, /// The value of the environment variable. - value: OsString, + value: String, } //-------------------------------------------------------------------------------------------------- @@ -52,16 +52,16 @@ impl EnvPair { /// * `var` - The name of the environment variable. /// * `value` - The value of the environment variable. /// - /// # Examples + /// ## Examples /// /// ``` /// use monocore::runtime::EnvPair; /// /// let env_pair = EnvPair::new("HOME", "/home/user"); - /// assert_eq!(env_pair.var().to_str().unwrap(), "HOME"); - /// assert_eq!(env_pair.value().to_str().unwrap(), "/home/user"); + /// assert_eq!(env_pair.get_var(), "HOME"); + /// assert_eq!(env_pair.get_value(), "/home/user"); /// ``` - pub fn new>(var: S, value: S) -> Self { + pub fn new>(var: S, value: S) -> Self { Self { var: var.into(), value: value.into(), @@ -92,12 +92,7 @@ impl FromStr for EnvPair { impl fmt::Display for EnvPair { /// Formats the environment variable pair following the format "=". fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "{}={}", - self.var.to_string_lossy(), - self.value.to_string_lossy() - ) + write!(f, "{}={}", self.var, self.value) } } @@ -131,19 +126,19 @@ mod tests { #[test] fn test_env_pair_new() { let env_pair = EnvPair::new("VAR", "VALUE"); - assert_eq!(env_pair.var, OsString::from("VAR")); - assert_eq!(env_pair.value, OsString::from("VALUE")); + assert_eq!(env_pair.var, String::from("VAR")); + assert_eq!(env_pair.value, String::from("VALUE")); } #[test] fn test_env_pair_from_str() -> anyhow::Result<()> { let env_pair: EnvPair = "VAR=VALUE".parse()?; - assert_eq!(env_pair.var, OsString::from("VAR")); - assert_eq!(env_pair.value, OsString::from("VALUE")); + assert_eq!(env_pair.var, String::from("VAR")); + assert_eq!(env_pair.value, String::from("VALUE")); let env_pair: EnvPair = "VAR=".parse()?; - assert_eq!(env_pair.var, OsString::from("VAR")); - assert_eq!(env_pair.value, OsString::from("")); + assert_eq!(env_pair.var, String::from("VAR")); + assert_eq!(env_pair.value, String::from("")); assert!("VAR".parse::().is_err()); assert!("=VALUE".parse::().is_err()); @@ -182,12 +177,12 @@ mod tests { #[test] fn test_env_pair_with_special_characters() -> anyhow::Result<()> { let env_pair: EnvPair = "VAR_WITH_UNDERSCORE=VALUE WITH SPACES".parse()?; - assert_eq!(env_pair.var, OsString::from("VAR_WITH_UNDERSCORE")); - assert_eq!(env_pair.value, OsString::from("VALUE WITH SPACES")); + assert_eq!(env_pair.get_var(), "VAR_WITH_UNDERSCORE"); + assert_eq!(env_pair.get_value(), "VALUE WITH SPACES"); let env_pair: EnvPair = "VAR.WITH.DOTS=VALUE_WITH_UNDERSCORE".parse()?; - assert_eq!(env_pair.var, OsString::from("VAR.WITH.DOTS")); - assert_eq!(env_pair.value, OsString::from("VALUE_WITH_UNDERSCORE")); + assert_eq!(env_pair.get_var(), "VAR.WITH.DOTS"); + assert_eq!(env_pair.get_value(), "VALUE_WITH_UNDERSCORE"); Ok(()) } diff --git a/monocore/lib/runtime/vm/rlimit.rs b/monocore/lib/runtime/vm/rlimit.rs index 89910b4..5d8b877 100644 --- a/monocore/lib/runtime/vm/rlimit.rs +++ b/monocore/lib/runtime/vm/rlimit.rs @@ -67,7 +67,7 @@ pub enum LinuxRLimitResource { /// The soft limit is the value that the kernel enforces for the corresponding resource. /// The hard limit acts as a ceiling for the soft limit. /// -/// # Examples +/// ## Examples /// /// ``` /// use monocore::runtime::{LinuxRlimit, LinuxRLimitResource}; @@ -75,19 +75,19 @@ pub enum LinuxRLimitResource { /// // Create a new resource limit for CPU time /// let cpu_limit = LinuxRlimit::new(LinuxRLimitResource::RLIMIT_CPU, 10, 20); /// -/// assert_eq!(cpu_limit.resource(), &LinuxRLimitResource::RLIMIT_CPU); -/// assert_eq!(cpu_limit.soft(), &10); -/// assert_eq!(cpu_limit.hard(), &20); +/// assert_eq!(cpu_limit.get_resource(), &LinuxRLimitResource::RLIMIT_CPU); +/// assert_eq!(cpu_limit.get_soft(), &10); +/// assert_eq!(cpu_limit.get_hard(), &20); /// /// // Parse a resource limit from a string /// let nofile_limit: LinuxRlimit = "RLIMIT_NOFILE=1000:2000".parse().unwrap(); /// -/// assert_eq!(nofile_limit.resource(), &LinuxRLimitResource::RLIMIT_NOFILE); -/// assert_eq!(nofile_limit.soft(), &1000); -/// assert_eq!(nofile_limit.hard(), &2000); +/// assert_eq!(nofile_limit.get_resource(), &LinuxRLimitResource::RLIMIT_NOFILE); +/// assert_eq!(nofile_limit.get_soft(), &1000); +/// assert_eq!(nofile_limit.get_hard(), &2000); /// ``` #[derive(Debug, Clone, PartialEq, Eq, Getters)] -#[getset(get = "pub")] +#[getset(get = "pub with_prefix")] pub struct LinuxRlimit { /// The resource to limit. resource: LinuxRLimitResource, @@ -123,15 +123,15 @@ impl LinuxRlimit { /// * `soft` - The soft limit value. /// * `hard` - The hard limit value. /// - /// # Examples + /// ## Examples /// /// ``` /// use monocore::runtime::{LinuxRlimit, LinuxRLimitResource}; /// /// let cpu_limit = LinuxRlimit::new(LinuxRLimitResource::RLIMIT_CPU, 10, 20); - /// assert_eq!(cpu_limit.resource(), &LinuxRLimitResource::RLIMIT_CPU); - /// assert_eq!(cpu_limit.soft(), &10); - /// assert_eq!(cpu_limit.hard(), &20); + /// assert_eq!(cpu_limit.get_resource(), &LinuxRLimitResource::RLIMIT_CPU); + /// assert_eq!(cpu_limit.get_soft(), &10); + /// assert_eq!(cpu_limit.get_hard(), &20); /// ``` pub fn new(resource: LinuxRLimitResource, soft: u64, hard: u64) -> Self { Self { diff --git a/monocore/lib/runtime/vm/vm.rs b/monocore/lib/runtime/vm/vm.rs index 68ec17a..fbbeccc 100644 --- a/monocore/lib/runtime/vm/vm.rs +++ b/monocore/lib/runtime/vm/vm.rs @@ -1,7 +1,7 @@ use std::{ffi::CString, path::PathBuf}; use getset::Getters; -use typed_path::UnixPathBuf; +use typed_path::Utf8UnixPathBuf; use crate::{ config::{PathPair, PortPair}, @@ -22,7 +22,7 @@ pub struct MicroVM { ctx_id: u32, /// The configuration for the microVM. - #[get = "pub"] + #[get = "pub with_prefix"] config: MicroVMConfig, } @@ -51,10 +51,10 @@ pub struct MicroVMConfig { pub rlimits: Vec, /// The working directory path to use for the microVM. - pub workdir_path: Option, + pub workdir_path: Option, /// The executable path to use for the microVM. - pub exec_path: Option, + pub exec_path: Option, /// The arguments to pass to the executable. pub argv: Vec, @@ -63,7 +63,7 @@ pub struct MicroVMConfig { pub env: Vec, /// The console output path to use for the microVM. - pub console_output: Option, + pub console_output: Option, } /// The log level to use for the microVM. @@ -144,8 +144,8 @@ impl MicroVM { // Add virtio-fs mounts for mount in &config.virtiofs { - let tag = CString::new(mount.guest().to_str().unwrap().as_bytes()).unwrap(); - let path = CString::new(mount.host().to_str().unwrap().as_bytes()).unwrap(); + let tag = CString::new(mount.get_guest().to_string().as_bytes()).unwrap(); + let path = CString::new(mount.get_host().to_string().as_bytes()).unwrap(); unsafe { let status = ffi::krun_add_virtiofs(ctx_id, tag.as_ptr(), path.as_ptr()); assert!(status >= 0, "Failed to add virtio-fs mount: {}", status); @@ -181,7 +181,7 @@ impl MicroVM { // Set working directory if let Some(workdir) = &config.workdir_path { - let c_workdir = CString::new(workdir.to_str().unwrap().as_bytes()).unwrap(); + let c_workdir = CString::new(workdir.to_string().as_bytes()).unwrap(); unsafe { let status = ffi::krun_set_workdir(ctx_id, c_workdir.as_ptr()); assert!(status >= 0, "Failed to set working directory: {}", status); @@ -190,7 +190,7 @@ impl MicroVM { // Set executable path, arguments, and environment variables if let Some(exec_path) = &config.exec_path { - let c_exec_path = CString::new(exec_path.to_str().unwrap().as_bytes()).unwrap(); + let c_exec_path = CString::new(exec_path.to_string().as_bytes()).unwrap(); let c_argv: Vec<_> = config .argv @@ -240,8 +240,7 @@ impl MicroVM { // Set console output if let Some(console_output) = &config.console_output { - let c_console_output = - CString::new(console_output.to_str().unwrap().as_bytes()).unwrap(); + let c_console_output = CString::new(console_output.to_string().as_bytes()).unwrap(); unsafe { let status = ffi::krun_set_console_output(ctx_id, c_console_output.as_ptr()); assert!(status >= 0, "Failed to set console output: {}", status); diff --git a/monofs/Cargo.toml b/monofs/Cargo.toml index b3ada66..0f77b9c 100644 --- a/monofs/Cargo.toml +++ b/monofs/Cargo.toml @@ -1,11 +1,27 @@ [package] name = "monofs" -version = "0.1.0" -edition = "2021" +version.workspace = true license.workspace = true +repository.workspace = true +authors.workspace = true +edition.workspace = true [lib] name = "monofs" path = "lib/lib.rs" [dependencies] +monoutils-store.workspace = true +serde.workspace = true +chrono = { workspace = true, features = ["serde"] } +getset.workspace = true +async-once-cell = "0.5.4" +anyhow.workspace = true +tokio.workspace = true +thiserror.workspace = true +futures.workspace = true +typed-path.workspace = true +bytes.workspace = true +aliasable = "0.1.3" +async-recursion = "1.1.1" +serde_json.workspace = true diff --git a/monofs/README.md b/monofs/README.md new file mode 100644 index 0000000..3a4a727 --- /dev/null +++ b/monofs/README.md @@ -0,0 +1,129 @@ +
+

monofs

+ +

+ + Build Status + + + License + +

+
+ +**`monofs`** is a powerful, distributed filesystem library designed for AI-driven sandboxed environments. It provides a simple and intuitive API for managing files and directories in a content-addressed storage system. + +## Features + +- Content-addressed storage +- Immutable data structures with copy-on-write semantics +- Support for files, directories, and symbolic links +- Asynchronous API for efficient I/O operations + +## Usage + +Here are some examples of how to use the `monofs` API: + +### Working with Files + +```rust +use monofs::filesystem::{File, FileInputStream, FileOutputStream}; +use monoutils_store::{MemoryStore, Storable}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let store = MemoryStore::default(); + + // Create a new file + let mut file = File::new(store.clone()); + + // Write content to the file + let mut output_stream = FileOutputStream::new(&mut file); + output_stream.write_all(b"Hello, monofs!").await?; + output_stream.shutdown().await?; + + // Read content from the file + let input_stream = FileInputStream::new(&file).await?; + let mut buffer = Vec::new(); + input_stream.read_to_end(&mut buffer).await?; + + println!("File content: {}", String::from_utf8_lossy(&buffer)); + + // Store the file + let file_cid = file.store().await?; + println!("Stored file with CID: {}", file_cid); + + Ok(()) +} +``` + +### Working with Directories + +```rust +use monofs::filesystem::{Dir, File, FsResult}; +use monoutils_store::{MemoryStore, Storable}; + +#[tokio::main] +async fn main() -> FsResult<()> { + let store = MemoryStore::default(); + + // Create a new root directory + let mut root = Dir::new(store.clone()); + + // Create a file in the directory + root.put_file("example.txt", File::new(store.clone()))?; + + // Create a subdirectory + root.put_dir("subdir", Dir::new(store.clone()))?; + + // List directory contents + for (name, entity) in root.get_entries() { + println!("- {}: {:?}", name, entity); + } + + // Store the directory + let root_cid = root.store().await?; + println!("Stored root directory with CID: {}", root_cid); + + Ok(()) +} +``` + +## API Overview + +- `File`: Represents a file in the filesystem +- `Dir`: Represents a directory in the filesystem +- `FileInputStream`: Provides read access to file contents +- `FileOutputStream`: Provides write access to file contents +- `Metadata`: Stores metadata for files and directories +- `Storable`: Trait for storing and loading entities from the content-addressed store + +For more detailed examples and API usage, check out the `examples` directory and the API documentation. + +## Development + +To set up `monofs` for development: + +1. Ensure you have Rust installed (latest stable version) +2. Clone the monocore repository: + ```sh + git clone https://github.com/appcypher/monocore + cd monocore/monofs + ``` +3. Build the project: + ```sh + cargo build + ``` +4. Run tests: + ```sh + cargo test + ``` + +## Contributing + +Contributions are welcome! Please read the [CONTRIBUTING.md](../CONTRIBUTING.md) file for guidelines on how to contribute to this project. + +## License + +This project is licensed under the [Apache License 2.0](../LICENSE). diff --git a/monofs/examples/dir_ops.rs b/monofs/examples/dir_ops.rs new file mode 100644 index 0000000..c50332d --- /dev/null +++ b/monofs/examples/dir_ops.rs @@ -0,0 +1,81 @@ +use monofs::filesystem::{Dir, File, FsResult}; +use monoutils_store::{MemoryStore, Storable}; + +//-------------------------------------------------------------------------------------------------- +// Function: main +//-------------------------------------------------------------------------------------------------- + +#[tokio::main] +async fn main() -> FsResult<()> { + // Create a new MemoryStore + let store = MemoryStore::default(); + + // Create a new root directory + let mut root = Dir::new(store.clone()); + println!("Created root directory: {:?}", root); + + // Find or create a file + let file = root.find_or_create("docs/readme.md", true).await?; + println!("Created file: {:?}", file); + + // Find or create a directory + let dir = root.find_or_create("projects/rust", false).await?; + println!("Created directory: {:?}", dir); + + // List contents of root directory + let entries = root.list()?; + println!("Root directory contents: {:?}", entries); + + // Copy a file + root.copy("docs/readme.md", "projects").await?; + println!("Copied 'readme.md' to 'projects' directory"); + + // Find the copied file + let copied_file = root.find("projects/readme.md").await?; + println!("Copied file: {:?}", copied_file); + + // Remove a file + let (removed_name, removed_entity) = root.remove("docs/readme.md").await?; + println!("Removed '{}': {:?}", removed_name, removed_entity); + + // Create and add a subdirectory + root.put_dir("subdir", Dir::new(store.clone()))?; + println!("Added subdirectory 'subdir'"); + + // Create and add a file to the root directory + root.put_file("example.txt", File::new(store.clone()))?; + println!("Added file 'example.txt' to root"); + + // List entries in the root directory + println!("Entries in root directory:"); + for (name, entity) in root.get_entries() { + println!("- {}: {:?}", name, entity); + } + + // Check if an entry exists + let file_exists = root.has_entry("example.txt").await?; + println!("'example.txt' exists: {}", file_exists); + + // Get and modify a subdirectory + if let Some(subdir) = root.get_dir_mut("subdir").await? { + subdir.put_file("subdir_file.txt", File::new(store.clone()))?; + println!("Added 'subdir_file.txt' to 'subdir'"); + } + + // Remove an entry + root.remove_entry("example.txt")?; + println!("Removed 'example.txt' from root"); + + // Check if the directory is empty + println!("Root directory is empty: {}", root.is_empty()); + + // Store the root directory + let root_cid = root.store().await?; + println!("Stored root directory with CID: {}", root_cid); + + // Load the root directory + let loaded_root = Dir::load(&root_cid, store).await?; + println!("Loaded root directory: {:?}", loaded_root); + + Ok(()) +} diff --git a/monofs/examples/file_ops.rs b/monofs/examples/file_ops.rs new file mode 100644 index 0000000..fd0b21e --- /dev/null +++ b/monofs/examples/file_ops.rs @@ -0,0 +1,57 @@ +use monofs::filesystem::{File, FileInputStream, FileOutputStream}; +use monoutils_store::{MemoryStore, Storable}; +use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; + +//-------------------------------------------------------------------------------------------------- +// Function: main +//-------------------------------------------------------------------------------------------------- + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // Create a new MemoryStore + let store = MemoryStore::default(); + + // Create a new file + let mut file = File::new(store.clone()); + println!("Created new file: {:?}", file); + + // Write content to the file + let content = b"Hello, monofs!"; + let mut output_stream = FileOutputStream::new(&mut file); + output_stream.write_all(content).await?; + output_stream.shutdown().await?; + println!("Wrote content to file"); + + // Read content from the file + let input_stream = FileInputStream::new(&file).await?; + let mut buffer = Vec::new(); + let mut reader = BufReader::new(input_stream); + reader.read_to_end(&mut buffer).await?; + println!( + "Read content from file: {}", + String::from_utf8_lossy(&buffer) + ); + drop(reader); // Drop reader to free up the input stream ref to the file + + // Check if the file is empty + println!("File is empty: {}", file.is_empty()); + + // Get and print file metadata + let metadata = file.get_metadata(); + println!("File metadata: {:?}", metadata); + + // Store the file + let file_cid = file.store().await?; + println!("Stored file with CID: {}", file_cid); + + // Load the file + let loaded_file = File::load(&file_cid, store).await?; + println!("Loaded file: {:?}", loaded_file); + + // Truncate the file + file.truncate(); + println!("Truncated file"); + println!("File is empty after truncation: {}", file.is_empty()); + + Ok(()) +} diff --git a/monofs/lib/config/default.rs b/monofs/lib/config/default.rs new file mode 100644 index 0000000..0021357 --- /dev/null +++ b/monofs/lib/config/default.rs @@ -0,0 +1,6 @@ +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +/// The default maximum depth of a softlink. +pub const DEFAULT_SOFTLINK_DEPTH: u32 = 10; diff --git a/monofs/lib/config/mod.rs b/monofs/lib/config/mod.rs new file mode 100644 index 0000000..5606c80 --- /dev/null +++ b/monofs/lib/config/mod.rs @@ -0,0 +1,9 @@ +//! Configuration types and helpers. + +mod default; + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +pub use default::*; diff --git a/monofs/lib/filesystem/dir/dir.rs b/monofs/lib/filesystem/dir/dir.rs new file mode 100644 index 0000000..9581885 --- /dev/null +++ b/monofs/lib/filesystem/dir/dir.rs @@ -0,0 +1,668 @@ +use std::{ + collections::{BTreeMap, HashMap}, + fmt::{self, Debug}, + str::FromStr, + sync::Arc, +}; + +use monoutils_store::{ + ipld::cid::Cid, IpldReferences, IpldStore, Storable, StoreError, StoreResult, +}; +use serde::{ + de::{self, DeserializeSeed}, + Deserialize, Deserializer, Serialize, +}; + +use crate::filesystem::{ + kind::EntityType, Entity, EntityCidLink, File, FsError, FsResult, Link, Metadata, SoftLink, +}; + +use super::Utf8UnixPathSegment; + +//-------------------------------------------------------------------------------------------------- +// Types: Dir +//-------------------------------------------------------------------------------------------------- + +/// Represents a directory node in the `monofs` _immutable_ file system. +/// +/// ## Important +/// +/// Entities in `monofs` are designed to be immutable and clone-on-write meaning writes create +/// forks of the entity. +#[derive(Clone)] +pub struct Dir +where + S: IpldStore, +{ + pub(super) inner: Arc>, +} + +#[derive(Clone)] +pub(super) struct DirInner +where + S: IpldStore, +{ + /// Directory metadata. + pub(crate) metadata: Metadata, + + /// The store used to persist blocks in the directory. + pub(crate) store: S, + + /// The entries in the directory. + pub(crate) entries: HashMap>, +} + +//-------------------------------------------------------------------------------------------------- +// Types: * +//-------------------------------------------------------------------------------------------------- + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct DirSerializable { + metadata: Metadata, + entries: BTreeMap, +} + +pub(crate) struct DirDeserializeSeed { + pub(crate) store: S, +} + +//-------------------------------------------------------------------------------------------------- +// Methods: Dir +//-------------------------------------------------------------------------------------------------- + +impl Dir +where + S: IpldStore, +{ + /// Creates a new directory with the given store. + /// + /// # Examples + /// + /// ``` + /// use monofs::filesystem::Dir; + /// use monoutils_store::MemoryStore; + /// + /// let store = MemoryStore::default(); + /// let dir = Dir::new(store); + /// + /// assert!(dir.is_empty()); + /// ``` + pub fn new(store: S) -> Self { + Self { + inner: Arc::new(DirInner { + metadata: Metadata::new(EntityType::Dir), + entries: HashMap::new(), + store, + }), + } + } + + /// Checks if an [`EntityCidLink`] with the given name exists in the directory. + /// + /// # Examples + /// + /// ``` + /// use monofs::filesystem::{Dir, Utf8UnixPathSegment}; + /// use monoutils_store::MemoryStore; + /// use monoutils_store::ipld::cid::Cid; + /// + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// let store = MemoryStore::default(); + /// let mut dir = Dir::new(store); + /// + /// let file_name = "example.txt"; + /// let file_cid: Cid = "bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".parse()?; + /// + /// dir.put_entry(file_name, file_cid.into())?; + /// + /// assert!(dir.has_entry(file_name).await?); + /// assert!(!dir.has_entry("nonexistent.txt").await?); + /// # Ok(()) + /// # } + /// ``` + pub async fn has_entry(&self, name: impl AsRef) -> FsResult { + let name = Utf8UnixPathSegment::from_str(name.as_ref())?; + Ok(self.inner.entries.contains_key(&name)) + } + + /// Adds a [`EntityCidLink`] and its associated name in the directory's entries. + /// + /// # Examples + /// + /// ``` + /// use monofs::filesystem::{Dir, Utf8UnixPathSegment}; + /// use monoutils_store::MemoryStore; + /// use monoutils_store::ipld::cid::Cid; + /// + /// # fn main() -> Result<(), Box> { + /// let store = MemoryStore::default(); + /// let mut dir = Dir::new(store); + /// + /// let file_name = "example.txt"; + /// let file_cid: Cid = "bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".parse()?; + /// + /// dir.put_entry(file_name, file_cid.into())?; + /// + /// assert!(dir.get_entry(file_name)?.is_some()); + /// # Ok(()) + /// # } + /// ``` + pub fn put_entry(&mut self, name: impl AsRef, link: EntityCidLink) -> FsResult<()> { + let name = Utf8UnixPathSegment::from_str(name.as_ref())?; + let inner = Arc::make_mut(&mut self.inner); + inner.entries.insert(name, link); + + Ok(()) + } + + /// Adds an [`Entity`] and its associated name in the directory's entries. + #[inline] + pub fn put_entity(&mut self, name: impl AsRef, entity: Entity) -> FsResult<()> + where + S: Send + Sync, + { + self.put_entry(name, EntityCidLink::from(entity)) + } + + /// Adds a [`Dir`] and its associated name in the directory's entries. + #[inline] + pub fn put_dir(&mut self, name: impl AsRef, dir: Dir) -> FsResult<()> + where + S: Send + Sync, + { + self.put_entry(name, EntityCidLink::from(dir)) + } + + /// Adds a [`File`] and its associated name in the directory's entries. + #[inline] + pub fn put_file(&mut self, name: impl AsRef, file: File) -> FsResult<()> + where + S: Send + Sync, + { + self.put_entry(name, EntityCidLink::from(file)) + } + + /// Adds a [`SoftLink`] and its associated name in the directory's entries. + #[inline] + pub fn put_softlink(&mut self, name: impl AsRef, softlink: SoftLink) -> FsResult<()> + where + S: Send + Sync, + { + self.put_entry(name, EntityCidLink::from(softlink)) + } + + /// Gets the [`EntityCidLink`] with the given name from the directory's entries. + /// + /// # Examples + /// + /// ``` + /// use monofs::filesystem::{Dir, Utf8UnixPathSegment}; + /// use monoutils_store::MemoryStore; + /// use monoutils_store::ipld::cid::Cid; + /// + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// let store = MemoryStore::default(); + /// let mut dir = Dir::new(store); + /// + /// let file_name = "example.txt"; + /// let file_cid: Cid = "bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".parse()?; + /// + /// dir.put_entry(file_name, file_cid.clone().into())?; + /// + /// let entry = dir.get_entry(file_name)?.unwrap(); + /// assert_eq!(entry.resolve_cid().await?, file_cid); + /// # Ok(()) + /// # } + /// ``` + #[inline] + pub fn get_entry(&self, name: impl AsRef) -> FsResult>> { + let name = Utf8UnixPathSegment::from_str(name.as_ref())?; + Ok(self.inner.entries.get(&name)) + } + + /// Gets the [`EntityCidLink`] with the given name from the directory's entries. + #[inline] + pub fn get_entry_mut( + &mut self, + name: impl AsRef, + ) -> FsResult>> { + let name = Utf8UnixPathSegment::from_str(name.as_ref())?; + let inner = Arc::make_mut(&mut self.inner); + Ok(inner.entries.get_mut(&name)) + } + + /// Gets the [`Entity`] with the associated name from the directory's entries. + pub async fn get_entity(&self, name: impl AsRef) -> FsResult>> + where + S: Send + Sync, + { + match self.get_entry(name)? { + Some(link) => Ok(Some(link.resolve_entity(self.inner.store.clone()).await?)), + None => Ok(None), + } + } + + /// Gets the [`Entity`] with the associated name from the directory's entries. + pub async fn get_entity_mut( + &mut self, + name: impl AsRef, + ) -> FsResult>> + where + S: Send + Sync, + { + let store = self.inner.store.clone(); + match self.get_entry_mut(name)? { + Some(link) => Ok(Some(link.resolve_entity_mut(store).await?)), + None => Ok(None), + } + } + + /// Gets the [`Dir`] with the associated name from the directory's entries. + pub async fn get_dir(&self, name: impl AsRef) -> FsResult>> + where + S: Send + Sync, + { + match self.get_entity(name).await? { + Some(Entity::Dir(dir)) => Ok(Some(dir)), + _ => Ok(None), + } + } + + /// Gets the [`Dir`] with the associated name from the directory's entries. + pub async fn get_dir_mut(&mut self, name: impl AsRef) -> FsResult>> + where + S: Send + Sync, + { + match self.get_entity_mut(name).await? { + Some(Entity::Dir(dir)) => Ok(Some(dir)), + _ => Ok(None), + } + } + + /// Gets the [`File`] with the associated name from the directory's entries. + pub async fn get_file(&self, name: impl AsRef) -> FsResult>> + where + S: Send + Sync, + { + match self.get_entity(name).await? { + Some(Entity::File(file)) => Ok(Some(file)), + _ => Ok(None), + } + } + + /// Gets the [`File`] with the associated name from the directory's entries. + pub async fn get_file_mut(&mut self, name: impl AsRef) -> FsResult>> + where + S: Send + Sync, + { + match self.get_entity_mut(name).await? { + Some(Entity::File(file)) => Ok(Some(file)), + _ => Ok(None), + } + } + + /// Gets the [`SoftLink`] with the associated name from the directory's entries. + pub async fn get_softlink(&self, name: impl AsRef) -> FsResult>> + where + S: Send + Sync, + { + match self.get_entity(name).await? { + Some(Entity::SoftLink(softlink)) => Ok(Some(softlink)), + _ => Ok(None), + } + } + + /// Gets the [`SoftLink`] with the associated name from the directory's entries. + pub async fn get_softlink_mut( + &mut self, + name: impl AsRef, + ) -> FsResult>> + where + S: Send + Sync, + { + match self.get_entity_mut(name).await? { + Some(Entity::SoftLink(softlink)) => Ok(Some(softlink)), + _ => Ok(None), + } + } + + /// Removes the [`EntityCidLink`] with the given name from the directory's entries. + pub fn remove_entry(&mut self, name: impl AsRef) -> FsResult> { + let name = Utf8UnixPathSegment::from_str(name.as_ref())?; + let inner = Arc::make_mut(&mut self.inner); + inner + .entries + .remove(&name) + .ok_or(FsError::PathNotFound(name.to_string())) + } + + /// Returns the metadata for the directory. + pub fn get_metadata(&self) -> &Metadata { + &self.inner.metadata + } + + /// Returns an iterator over the entries in the directory. + pub fn get_entries(&self) -> impl Iterator)> { + self.inner.entries.iter() + } + + /// Returns the store used to persist the file. + pub fn get_store(&self) -> &S { + &self.inner.store + } + + /// Returns `true` if the directory is empty. + /// + /// # Examples + /// + /// ``` + /// use monofs::filesystem::{Dir, Utf8UnixPathSegment}; + /// use monoutils_store::MemoryStore; + /// use monoutils_store::ipld::cid::Cid; + /// + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// let store = MemoryStore::default(); + /// let mut dir = Dir::new(store); + /// + /// assert!(dir.is_empty()); + /// + /// let file_name = "example.txt"; + /// let file_cid: Cid = "bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".parse()?; + /// + /// dir.put_entry(file_name, file_cid.into())?; + /// + /// assert!(!dir.is_empty()); + /// # Ok(()) + /// # } + /// ``` + pub fn is_empty(&self) -> bool { + self.inner.entries.is_empty() + } + + /// Deserializes to a `Dir` using an arbitrary deserializer and store. + pub fn deserialize_with<'de>( + deserializer: impl Deserializer<'de, Error: Into>, + store: S, + ) -> FsResult { + DirDeserializeSeed::new(store) + .deserialize(deserializer) + .map_err(Into::into) + } + + /// Tries to create a new `Dir` from a serializable representation. + pub(crate) fn try_from_serializable(serializable: DirSerializable, store: S) -> FsResult { + let entries: HashMap<_, _> = serializable + .entries + .into_iter() + .map(|(segment, cid)| Ok((segment.parse()?, Link::from(cid)))) + .collect::>()?; + + Ok(Dir { + inner: Arc::new(DirInner { + metadata: serializable.metadata, + store, + entries, + }), + }) + } +} + +//-------------------------------------------------------------------------------------------------- +// Methods: DirDeserializeSeed +//-------------------------------------------------------------------------------------------------- + +impl DirDeserializeSeed { + fn new(store: S) -> Self { + Self { store } + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Storable for Dir +where + S: IpldStore + Send + Sync, +{ + async fn store(&self) -> StoreResult { + let mut entries = BTreeMap::new(); + for (k, v) in self.get_entries() { + entries.insert( + k.to_string(), + v.resolve_cid().await.map_err(StoreError::custom)?, + ); + } + + let serializable = DirSerializable { + metadata: self.inner.metadata.clone(), + entries, + }; + + self.inner.store.put_node(&serializable).await + } + + async fn load(cid: &Cid, store: S) -> StoreResult { + let serializable: DirSerializable = store.get_node(cid).await?; + Dir::try_from_serializable(serializable, store).map_err(StoreError::custom) + } +} + +impl Debug for Dir +where + S: IpldStore, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Dir") + .field("metadata", &self.inner.metadata) + .field( + "entries", + &self + .get_entries() + .map(|(_, v)| v.get_cid()) // TODO: Resolve value here. + .collect::>(), + ) + .finish() + } +} + +impl<'de, S> DeserializeSeed<'de> for DirDeserializeSeed +where + S: IpldStore, +{ + type Value = Dir; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let serializable = DirSerializable::deserialize(deserializer)?; + Dir::try_from_serializable(serializable, self.store).map_err(de::Error::custom) + } +} + +impl IpldReferences for DirSerializable { + fn get_references<'a>(&'a self) -> Box + Send + 'a> { + Box::new(self.entries.values()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use anyhow::Ok; + use monoutils_store::MemoryStore; + + use crate::{config::DEFAULT_SOFTLINK_DEPTH, filesystem::SyncType}; + + use super::*; + + #[tokio::test] + async fn test_dir_constructor() -> anyhow::Result<()> { + let store = MemoryStore::default(); + let dir = Dir::new(store); + + assert!(dir.inner.entries.is_empty()); + + Ok(()) + } + + #[tokio::test] + async fn test_dir_put_get_entries() -> anyhow::Result<()> { + let mut dir = Dir::new(MemoryStore::default()); + + let file1_cid: Cid = + "bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".parse()?; + let file2_cid: Cid = + "bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".parse()?; + + let file1_name = "file1"; + let file2_name = "file2"; + + dir.put_entry(file1_name, file1_cid.into())?; + dir.put_entry(file2_name, file2_cid.into())?; + + assert_eq!(dir.inner.entries.len(), 2); + assert_eq!( + dir.get_entry(&file1_name)?.unwrap().get_cid(), + Some(&file1_cid) + ); + assert_eq!( + dir.get_entry(&file2_name)?.unwrap().get_cid(), + Some(&file2_cid) + ); + + Ok(()) + } + + #[tokio::test] + async fn test_dir_stores_loads() -> anyhow::Result<()> { + let store = MemoryStore::default(); + let mut dir = Dir::new(store.clone()); + + let file_name = "file1"; + let file_cid: Cid = + "bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".parse()?; + + dir.put_entry(file_name, file_cid.into())?; + + let cid = dir.store().await?; + let loaded_dir = Dir::load(&cid, store.clone()).await?; + + // Assert that the metadata is the same + assert_eq!(dir.get_metadata(), loaded_dir.get_metadata()); + + // Assert that the number of entries is the same + assert_eq!(dir.get_entries().count(), loaded_dir.get_entries().count()); + + // Assert that the entry we added exists in the loaded directory + let loaded_entry = loaded_dir + .get_entry(&file_name)? + .expect("Entry should exist"); + + assert_eq!(loaded_entry.get_cid(), Some(&file_cid)); + + Ok(()) + } + + #[tokio::test] + async fn test_dir_has_entry() -> anyhow::Result<()> { + let mut dir = Dir::new(MemoryStore::default()); + let file_name = "example.txt"; + let file_cid: Cid = + "bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".parse()?; + + dir.put_entry(file_name, file_cid.into())?; + + assert!(dir.has_entry(file_name).await?); + assert!(!dir.has_entry("nonexistent.txt").await?); + + Ok(()) + } + + #[tokio::test] + async fn test_dir_remove_entry() -> anyhow::Result<()> { + let mut dir = Dir::new(MemoryStore::default()); + let file_name = "example.txt"; + let file_cid: Cid = + "bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".parse()?; + + dir.put_entry(file_name, file_cid.clone().into())?; + assert!(dir.has_entry(file_name).await?); + + let removed_entry = dir.remove_entry(file_name)?; + assert_eq!(removed_entry.get_cid(), Some(&file_cid)); + assert!(!dir.has_entry(file_name).await?); + + assert!(dir.remove_entry("nonexistent.txt").is_err()); + + Ok(()) + } + + #[tokio::test] + async fn test_dir_get_metadata() -> anyhow::Result<()> { + let dir = Dir::new(MemoryStore::default()); + let metadata = dir.get_metadata(); + + assert_eq!(*metadata.get_entity_type(), EntityType::Dir); + assert_eq!(*metadata.get_sync_type(), SyncType::RAFT); + assert_eq!(*metadata.get_softlink_depth(), DEFAULT_SOFTLINK_DEPTH); + + Ok(()) + } + + #[tokio::test] + async fn test_dir_is_empty() -> anyhow::Result<()> { + let mut dir = Dir::new(MemoryStore::default()); + assert!(dir.is_empty()); + + let file_name = "example.txt"; + let file_cid: Cid = + "bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".parse()?; + + dir.put_entry(file_name, file_cid.into())?; + assert!(!dir.is_empty()); + + dir.remove_entry(file_name)?; + assert!(dir.is_empty()); + + Ok(()) + } + + #[tokio::test] + async fn test_dir_get_entries() -> anyhow::Result<()> { + let mut dir = Dir::new(MemoryStore::default()); + let file1_name = "file1.txt"; + let file2_name = "file2.txt"; + let file1_cid: Cid = + "bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".parse()?; + let file2_cid: Cid = + "bafkreihwsnuregceqh263vgdaatvch6micl2phrh2tdwkaqsch7jpo5nuu".parse()?; + + dir.put_entry(file1_name, file1_cid.clone().into())?; + dir.put_entry(file2_name, file2_cid.clone().into())?; + + let entries: Vec<_> = dir.get_entries().collect(); + assert_eq!(entries.len(), 2); + + let entry1 = entries + .iter() + .find(|(name, _)| name.as_str() == file1_name) + .unwrap(); + let entry2 = entries + .iter() + .find(|(name, _)| name.as_str() == file2_name) + .unwrap(); + + assert_eq!(entry1.1.get_cid(), Some(&file1_cid)); + assert_eq!(entry2.1.get_cid(), Some(&file2_cid)); + + Ok(()) + } +} diff --git a/monofs/lib/filesystem/dir/find.rs b/monofs/lib/filesystem/dir/find.rs new file mode 100644 index 0000000..a009449 --- /dev/null +++ b/monofs/lib/filesystem/dir/find.rs @@ -0,0 +1,385 @@ +use std::fmt::Debug; + +use monoutils_store::IpldStore; +use typed_path::{Utf8UnixComponent, Utf8UnixPath}; + +use crate::filesystem::{entity::Entity, FsError, FsResult}; + +use super::{Dir, Utf8UnixPathSegment}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Result type for `find_dir*` functions. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FindResult { + /// The directory was found. + Found { + /// The directory containing the entity. + dir: T, + }, + + /// The directory was not found. + NotFound { + /// The last found directory in the path. + dir: T, + + /// The depth of the path to the entity. + depth: usize, + }, + + /// Intermediate path is not a directory. + NotADir { + /// The depth of the path to the entity. + depth: usize, + }, +} + +/// Result type for `find_dir` function. +pub type FindResultDir<'a, S> = FindResult<&'a Dir>; + +/// Result type for `find_dir_mut` function. +pub type FindResultDirMut<'a, S> = FindResult<&'a mut Dir>; + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Looks for a directory at the specified path. +/// +/// This function navigates through the directory structure starting from the given directory, +/// following the path specified by `path`. It attempts to resolve each component of the path +/// until it either finds the target directory, encounters an error, or determines that the path +/// is not found or invalid. +/// +/// ## Examples +/// +/// ``` +/// use monofs::filesystem::{Dir, find_dir}; +/// use monoutils_store::MemoryStore; +/// +/// # #[tokio::main] +/// # async fn main() -> anyhow::Result<()> { +/// let store = MemoryStore::default(); +/// let root_dir = Dir::new(store); +/// let result = find_dir(&root_dir, "some/path/to/entity").await?; +/// # Ok(()) +/// # } +/// ``` +/// +/// ## Note +/// +/// The function does not support the following path components: +/// - `.` +/// - `..` +/// - `/` +/// +/// If any of these components are present in the path, the function will return an error. +pub async fn find_dir(mut dir: &Dir, path: impl AsRef) -> FsResult> +where + S: IpldStore + Send + Sync, +{ + let path = Utf8UnixPath::new(path.as_ref()); + + // Convert path components to Utf8UnixPathSegment and collect them + let components = path + .components() + .map(|ref c| match c { + Utf8UnixComponent::RootDir => Err(FsError::InvalidSearchPath(path.to_string())), + Utf8UnixComponent::CurDir => Err(FsError::InvalidSearchPath(path.to_string())), + Utf8UnixComponent::ParentDir => Err(FsError::InvalidSearchPath(path.to_string())), + _ => Ok(Utf8UnixPathSegment::try_from(c)?), + }) + .collect::, _>>()?; + + // Process intermediate components (if any) + for (depth, segment) in components.iter().enumerate() { + match dir.get_entity(segment).await? { + Some(Entity::Dir(d)) => { + dir = d; + } + Some(Entity::SoftLink(_)) => { + // SoftLinks are not supported yet, so we return an error + return Err(FsError::SoftLinkNotSupportedYet(components.clone())); + } + Some(_) => { + // If we encounter a non-directory entity in the middle of the path, + // we return NotADir result + return Ok(FindResult::NotADir { depth }); + } + None => { + // If an intermediate component doesn't exist, + // we return NotFound result + return Ok(FindResult::NotFound { dir, depth }); + } + } + } + + Ok(FindResult::Found { dir }) +} + +/// Looks for a directory at the specified path. This is a mutable version of `find_dir`. +/// +/// This function navigates through the directory structure starting from the given directory, +/// following the path specified by `path`. It attempts to resolve each component of the path +/// until it either finds the target directory, encounters an error, or determines that the path +/// is not found or invalid. +/// +/// ## Examples +/// +/// ``` +/// use monofs::filesystem::{Dir, find_dir_mut}; +/// use monoutils_store::MemoryStore; +/// +/// # #[tokio::main] +/// # async fn main() -> anyhow::Result<()> { +/// let store = MemoryStore::default(); +/// let mut root_dir = Dir::new(store); +/// let result = find_dir_mut(&mut root_dir, "some/path/to/entity").await?; +/// # Ok(()) +/// # } +/// ``` +/// +/// ## Note +/// +/// The function does not support the following path components: +/// - `.` +/// - `..` +/// - `/` +/// +/// If any of these components are present in the path, the function will return an error. +pub async fn find_dir_mut( + mut dir: &mut Dir, + path: impl AsRef, +) -> FsResult> +where + S: IpldStore + Send + Sync, +{ + let path = Utf8UnixPath::new(path.as_ref()); + + // Convert path components to Utf8UnixPathSegment and collect them + let components = path + .components() + .map(|ref c| match c { + Utf8UnixComponent::RootDir => Err(FsError::InvalidSearchPath(path.to_string())), + Utf8UnixComponent::CurDir => Err(FsError::InvalidSearchPath(path.to_string())), + Utf8UnixComponent::ParentDir => Err(FsError::InvalidSearchPath(path.to_string())), + _ => Ok(Utf8UnixPathSegment::try_from(c)?), + }) + .collect::, _>>()?; + + // Process intermediate components (if any) + for (depth, segment) in components.iter().enumerate() { + match dir.get_entity(segment).await? { + Some(Entity::Dir(_)) => { + // A hack to get a mutable reference to the directory + dir = dir.get_dir_mut(segment).await?.unwrap(); + } + Some(Entity::SoftLink(_)) => { + // SoftLinks are not supported yet, so we return an error + return Err(FsError::SoftLinkNotSupportedYet(components.clone())); + } + Some(_) => { + // If we encounter a non-directory entity in the middle of the path, + // we return NotADir result + return Ok(FindResult::NotADir { depth }); + } + None => { + // If an intermediate component doesn't exist, + // we return NotFound result + return Ok(FindResult::NotFound { dir, depth }); + } + } + } + + Ok(FindResult::Found { dir }) +} + +/// Retrieves an existing entity or creates a new one at the specified path. +/// +/// This function checks the existence of an entity at the given path. If the entity +/// exists, it returns the entity. If the entity does not exist, it creates a new +/// directory hierarchy and returns the new entity. +/// +/// ## Examples +/// +/// ``` +/// use monofs::filesystem::{Dir, find_or_create_dir}; +/// use monoutils_store::MemoryStore; +/// +/// # #[tokio::main] +/// # async fn main() -> anyhow::Result<()> { +/// let store = MemoryStore::default(); +/// let mut root_dir = Dir::new(store); +/// let new_dir = find_or_create_dir(&mut root_dir, "new/nested/directory").await?; +/// assert!(new_dir.is_empty()); +/// # Ok(()) +/// # } +/// ``` +pub async fn find_or_create_dir(dir: &mut Dir, path: impl AsRef) -> FsResult<&mut Dir> +where + S: IpldStore + Send + Sync, +{ + let path = Utf8UnixPath::new(path.as_ref()); + + match find_dir_mut(dir, path).await { + Ok(FindResult::Found { dir }) => Ok(dir), + Ok(FindResult::NotFound { mut dir, depth }) => { + for component in path.components().skip(depth) { + let new_dir = Dir::new(dir.get_store().clone()); + let segment = Utf8UnixPathSegment::try_from(&component)?; + + dir.put_dir(segment.clone(), new_dir)?; + dir = dir.get_dir_mut(&segment).await?.unwrap(); + } + + Ok(dir) + } + Ok(FindResult::NotADir { depth }) => { + let components = path + .components() + .take(depth + 1) + .map(|c| c.to_string()) + .collect::>(); + + Err(FsError::NotADirectory(components.join("/"))) + } + Err(e) => Err(e), + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use monoutils_store::MemoryStore; + + use crate::filesystem::File; + + use super::*; + + mod fixtures { + use monoutils_store::Storable; + + use super::*; + + pub(super) async fn setup_test_filesystem() -> anyhow::Result> { + let store = MemoryStore::default(); + let mut root = Dir::new(store.clone()); + + let mut subdir1 = Dir::new(store.clone()); + let mut subdir2 = Dir::new(store.clone()); + + let file1 = File::new(store.clone()); + let file2 = File::new(store.clone()); + + let file1_cid = file1.store().await?; + subdir1.put_entry("file1.txt", file1_cid.into())?; + + let file2_cid = file2.store().await?; + subdir2.put_entry("file2.txt", file2_cid.into())?; + + let subdir2_cid = subdir2.store().await?; + subdir1.put_entry("subdir2", subdir2_cid.into())?; + + let subdir1_cid = subdir1.store().await?; + root.put_entry("subdir1", subdir1_cid.into())?; + + Ok(root) + } + } + + #[tokio::test] + async fn test_find_dir() -> anyhow::Result<()> { + let root = fixtures::setup_test_filesystem().await?; + + // Test finding existing directories + let result = find_dir(&root, "subdir1").await?; + assert!(matches!(result, FindResult::Found { .. })); + + let result = find_dir(&root, "subdir1/subdir2").await?; + assert!(matches!(result, FindResult::Found { .. })); + + // Test finding non-existent directories + let result = find_dir(&root, "nonexistent").await?; + assert!(matches!(result, FindResult::NotFound { depth: 0, .. })); + + let result = find_dir(&root, "subdir1/nonexistent").await?; + assert!(matches!(result, FindResult::NotFound { depth: 1, .. })); + + // Test finding a path that contains a file + let result = find_dir(&root, "subdir1/file1.txt/invalid").await?; + assert!(matches!(result, FindResult::NotADir { depth: 1 })); + + // Test invalid paths + let result = find_dir(&root, "/invalid/path").await; + assert!(matches!(result, Err(FsError::InvalidSearchPath(_)))); + + let result = find_dir(&root, "invalid/../path").await; + assert!(matches!(result, Err(FsError::InvalidSearchPath(_)))); + + let result = find_dir(&root, "./invalid/path").await; + assert!(matches!(result, Err(FsError::InvalidSearchPath(_)))); + + Ok(()) + } + + #[tokio::test] + async fn test_find_dir_mut() -> anyhow::Result<()> { + let mut root = fixtures::setup_test_filesystem().await?; + + // Test finding existing directories + let result = find_dir_mut(&mut root, "subdir1").await?; + assert!(matches!(result, FindResult::Found { .. })); + + let result = find_dir_mut(&mut root, "subdir1/subdir2").await?; + assert!(matches!(result, FindResult::Found { .. })); + + // Test finding non-existent directories + let result = find_dir_mut(&mut root, "nonexistent").await?; + assert!(matches!(result, FindResult::NotFound { depth: 0, .. })); + + let result = find_dir_mut(&mut root, "subdir1/nonexistent").await?; + assert!(matches!(result, FindResult::NotFound { depth: 1, .. })); + + // Test finding a path that contains a file + let result = find_dir_mut(&mut root, "subdir1/file1.txt/invalid").await?; + assert!(matches!(result, FindResult::NotADir { depth: 1 })); + + Ok(()) + } + + #[tokio::test] + async fn test_find_or_create_dir() -> anyhow::Result<()> { + let mut root = fixtures::setup_test_filesystem().await?; + + // Test creating a new directory + let new_dir = find_or_create_dir(&mut root, "new_dir").await?; + assert!(new_dir.is_empty()); + + // Verify the new directory exists + let result = find_dir(&root, "new_dir").await?; + assert!(matches!(result, FindResult::Found { .. })); + + // Test creating a nested structure + let nested_dir = find_or_create_dir(&mut root, "parent/child/grandchild").await?; + assert!(nested_dir.is_empty()); + + // Verify the nested structure exists + let result = find_dir(&root, "parent/child/grandchild").await?; + assert!(matches!(result, FindResult::Found { .. })); + + // Test getting an existing directory + let existing_dir = find_or_create_dir(&mut root, "subdir1").await?; + assert!(!existing_dir.is_empty()); + + // Test creating a directory where a file already exists + let result = find_or_create_dir(&mut root, "subdir1/file1.txt").await; + assert!(matches!(result, Err(FsError::NotADirectory(_)))); + + Ok(()) + } +} diff --git a/monofs/lib/filesystem/dir/mod.rs b/monofs/lib/filesystem/dir/mod.rs new file mode 100644 index 0000000..61524c7 --- /dev/null +++ b/monofs/lib/filesystem/dir/mod.rs @@ -0,0 +1,14 @@ +//! Directory implementation. + +mod dir; +mod find; +mod ops; +mod segment; + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +pub use dir::*; +pub use find::*; +pub use segment::*; diff --git a/monofs/lib/filesystem/dir/ops.rs b/monofs/lib/filesystem/dir/ops.rs new file mode 100644 index 0000000..a60ac40 --- /dev/null +++ b/monofs/lib/filesystem/dir/ops.rs @@ -0,0 +1,612 @@ +use monoutils_store::IpldStore; +use typed_path::Utf8UnixPath; + +use crate::{ + filesystem::{dir::find, entity::Entity, file::File, EntityCidLink, FsError, FsResult}, + utils::path, +}; + +use super::{Dir, FindResult, Utf8UnixPathSegment}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Directory operations. +impl Dir +where + S: IpldStore + Send + Sync, +{ + /// Finds an entity in the directory structure given a path. + /// + /// This method traverses the directory structure to find the entity specified by the path. + /// It returns a reference to the found entity if it exists. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::{Dir, Entity, FsResult}; + /// use monoutils_store::MemoryStore; + /// + /// # #[tokio::main] + /// # async fn main() -> FsResult<()> { + /// let mut dir = Dir::new(MemoryStore::default()); + /// dir.find_or_create("foo/bar.txt", true).await?; + /// + /// let entity = dir.find("foo/bar.txt").await?; + /// assert!(matches!(entity, Some(Entity::File(_)))); + /// # Ok(()) + /// # } + /// ``` + pub async fn find(&self, path: impl AsRef) -> FsResult>> { + let path = Utf8UnixPath::new(path.as_ref()); + + if path.has_root() { + return Err(FsError::PathHasRoot(path.to_string())); + } + + let (parent, file_name) = path::split_last(path)?; + if let Some(parent_path) = parent { + return match find::find_dir(self, parent_path).await? { + FindResult::Found { dir } => dir.get_entity(&file_name).await, + _ => Ok(None), + }; + } + + self.get_entity(&file_name).await + } + + /// Finds an entity in the directory structure given a path, returning a mutable reference. + /// + /// This method is similar to `find`, but it returns a mutable reference to the found entity. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::{Dir, Entity, FsResult}; + /// use monoutils_store::MemoryStore; + /// + /// # #[tokio::main] + /// # async fn main() -> FsResult<()> { + /// let mut dir = Dir::new(MemoryStore::default()); + /// dir.find_or_create("foo/bar.txt", true).await?; + /// + /// let entity = dir.find_mut("foo/bar.txt").await?; + /// assert!(matches!(entity, Some(Entity::File(_)))); + /// # Ok(()) + /// # } + /// ``` + pub async fn find_mut(&mut self, path: impl AsRef) -> FsResult>> { + let path = Utf8UnixPath::new(path.as_ref()); + + if path.has_root() { + return Err(FsError::PathHasRoot(path.to_string())); + } + + let (parent, file_name) = path::split_last(path)?; + if let Some(parent_path) = parent { + return match find::find_dir_mut(self, parent_path).await? { + FindResult::Found { dir } => dir.get_entity_mut(&file_name).await, + _ => Ok(None), + }; + } + + self.get_entity_mut(&file_name).await + } + + /// Finds an entity in the directory structure or creates it if it doesn't exist. + /// + /// This method traverses the directory structure to find the entity specified by the path. + /// If the entity doesn't exist, it creates a new file or directory based on the `file` parameter. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::{Dir, Entity, FsResult}; + /// use monoutils_store::MemoryStore; + /// + /// # #[tokio::main] + /// # async fn main() -> FsResult<()> { + /// let mut dir = Dir::new(MemoryStore::default()); + /// + /// // Create a file + /// let file = dir.find_or_create("foo/bar.txt", true).await?; + /// assert!(matches!(file, Entity::File(_))); + /// + /// // Create a directory + /// let subdir = dir.find_or_create("baz", false).await?; + /// assert!(matches!(subdir, Entity::Dir(_))); + /// # Ok(()) + /// # } + /// ``` + pub async fn find_or_create( + &mut self, + path: impl AsRef, + file: bool, + ) -> FsResult<&mut Entity> { + let path = Utf8UnixPath::new(path.as_ref()); + + if path.has_root() { + return Err(FsError::PathHasRoot(path.to_string())); + } + + let (parent, file_name) = path::split_last(path)?; + let parent_dir = match parent { + Some(parent_path) => find::find_or_create_dir(self, parent_path).await?, + None => self, + }; + + if parent_dir.has_entry(&file_name).await? { + return parent_dir + .get_entity_mut(&file_name) + .await? + .ok_or_else(|| FsError::PathNotFound(path.to_string())); + } + + let new_entity = if file { + Entity::File(File::new(parent_dir.get_store().clone())) + } else { + Entity::Dir(Dir::new(parent_dir.get_store().clone())) + }; + + parent_dir.put_entity(file_name.clone(), new_entity)?; + + parent_dir + .get_entity_mut(&file_name) + .await? + .ok_or_else(|| FsError::PathNotFound(path.to_string())) + } + + /// Lists all entries in the current directory. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::{Dir, FsResult}; + /// use monoutils_store::MemoryStore; + /// + /// # #[tokio::main] + /// # async fn main() -> FsResult<()> { + /// let mut dir = Dir::new(MemoryStore::default()); + /// dir.find_or_create("foo", false).await?; + /// dir.find_or_create("bar.txt", true).await?; + /// + /// let entries = dir.list()?; + /// assert_eq!(entries.len(), 2); + /// assert!(entries.contains(&"foo".parse()?)); + /// assert!(entries.contains(&"bar.txt".parse()?)); + /// # Ok(()) + /// # } + /// ``` + pub fn list(&self) -> FsResult> { + Ok(self.inner.entries.keys().cloned().collect()) + } + + /// Copies an entity from the source path to the target **directory**. + /// + /// The target path must be a directory. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::{Dir, Entity, FsResult}; + /// use monoutils_store::MemoryStore; + /// + /// # #[tokio::main] + /// # async fn main() -> FsResult<()> { + /// let mut dir = Dir::new(MemoryStore::default()); + /// dir.find_or_create("source/file.txt", true).await?; + /// dir.find_or_create("target", false).await?; + /// + /// dir.copy("source/file.txt", "target").await?; + /// + /// assert!(dir.find("target/file.txt").await?.is_some()); + /// # Ok(()) + /// # } + /// ``` + pub async fn copy(&mut self, source: impl AsRef, target: impl AsRef) -> FsResult<()> { + let source = Utf8UnixPath::new(source.as_ref()); + let target = Utf8UnixPath::new(target.as_ref()); + + if source.has_root() || target.has_root() { + return Err(FsError::PathHasRoot(source.to_string())); + } + + let (source_parent, source_filename) = path::split_last(source)?; + + // Find source parent directory and entity + let source_entity = if let Some(parent_path) = source_parent { + let parent_dir = self + .find(parent_path) + .await? + .and_then(|entity| { + if let Entity::Dir(dir) = entity { + Some(dir) + } else { + None + } + }) + .ok_or_else(|| FsError::SourceIsNotADir(parent_path.to_string()))?; + + parent_dir + .get_entity(&source_filename) + .await? + .cloned() + .ok_or_else(|| FsError::PathNotFound(source.to_string()))? + } else { + self.get_entity(&source_filename) + .await? + .cloned() + .ok_or_else(|| FsError::PathNotFound(source.to_string()))? + }; + + // Find target directory + let target_dir = match self.find_mut(target).await? { + Some(Entity::Dir(dir)) => dir, + _ => return Err(FsError::TargetIsNotADir(target.to_string())), + }; + + // Copy entity to target directory + target_dir.put_entity(source_filename, source_entity)?; + + Ok(()) + } + + /// Moves an entity from the source path to the target directory. + /// + /// The target path must be a directory. + pub async fn r#move( + &mut self, + _source: impl AsRef, + _target: impl AsRef, + ) -> FsResult<()> { + todo!("coming soon! `move` is tricky and needs to be handled properly") + } + + /// Alias for `r#move`. + #[inline] + pub async fn mv(&mut self, source: impl AsRef, target: impl AsRef) -> FsResult<()> { + self.r#move(source, target).await + } + + /// Removes an entity at the specified path and returns it. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::{Dir, Entity, FsResult}; + /// use monoutils_store::MemoryStore; + /// + /// # #[tokio::main] + /// # async fn main() -> FsResult<()> { + /// let mut dir = Dir::new(MemoryStore::default()); + /// dir.find_or_create("foo/bar.txt", true).await?; + /// + /// let (filename, _entity) = dir.remove("foo/bar.txt").await?; + /// + /// assert_eq!(filename, "bar.txt".parse()?); + /// assert!(dir.find("foo/bar.txt").await?.is_none()); + /// # Ok(()) + /// # } + /// ``` + pub async fn remove( + &mut self, + path: impl AsRef, + ) -> FsResult<(Utf8UnixPathSegment, EntityCidLink)> { + let path = Utf8UnixPath::new(path.as_ref()); + + if path.has_root() { + return Err(FsError::PathHasRoot(path.to_string())); + } + + let (parent, filename) = path::split_last(path)?; + + let parent_dir = if let Some(parent_path) = parent { + self.find_mut(parent_path) + .await? + .and_then(|entity| { + if let Entity::Dir(dir) = entity { + Some(dir) + } else { + None + } + }) + .ok_or_else(|| FsError::SourceIsNotADir(parent_path.to_string()))? + } else { + self + }; + + let entity = parent_dir.remove_entry(&filename)?; + + Ok((filename, entity)) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use monoutils_store::{MemoryStore, Storable}; + + #[tokio::test] + async fn test_ops_find() -> FsResult<()> { + let mut dir = Dir::new(MemoryStore::default()); + + // Create a file and a subdirectory + dir.find_or_create("foo/bar.txt", true).await?; + dir.find_or_create("baz", false).await?; + + // Test finding existing entities + assert!(matches!( + dir.find("foo/bar.txt").await?, + Some(Entity::File(_)) + )); + assert!(matches!(dir.find("baz").await?, Some(Entity::Dir(_)))); + + // Test finding non-existent entity + assert!(dir.find("nonexistent").await?.is_none()); + + Ok(()) + } + + #[tokio::test] + async fn test_ops_find_mut() -> FsResult<()> { + let mut dir = Dir::new(MemoryStore::default()); + + // Create a file and a subdirectory + dir.find_or_create("foo/bar.txt", true).await?; + dir.find_or_create("baz", false).await?; + + // Test finding existing entities mutably + assert!(matches!( + dir.find_mut("foo/bar.txt").await?, + Some(Entity::File(_)) + )); + assert!(matches!(dir.find_mut("baz").await?, Some(Entity::Dir(_)))); + + // Test finding non-existent entity + assert!(dir.find_mut("nonexistent").await?.is_none()); + + // Test modifying a found file + if let Some(Entity::File(file)) = dir.find_mut("foo/bar.txt").await? { + let content = "Hello, World!".as_bytes(); + let content_cid = file.get_store().put_raw_block(content).await?; + file.set_content(Some(content_cid)); + file.store().await?; + assert_eq!(file.get_content(), Some(&content_cid)); + } else { + panic!("Expected to find a file"); + } + + // Verify the modification persists + if let Some(Entity::File(file)) = dir.find("foo/bar.txt").await? { + let content_cid = file.get_content().expect("File should have content"); + let content = file.get_store().get_raw_block(content_cid).await?; + assert_eq!(content, "Hello, World!".as_bytes()); + } else { + panic!("Expected to find a file"); + } + + Ok(()) + } + + #[tokio::test] + async fn test_ops_find_or_create() -> FsResult<()> { + let mut dir = Dir::new(MemoryStore::default()); + + // Create a file + assert!(dir.find("foo/bar.txt").await?.is_none()); + let file = dir.find_or_create("foo/bar.txt", true).await?; + assert!(matches!(file, Entity::File(_))); + + // Create a directory + assert!(dir.find("baz").await?.is_none()); + let subdir = dir.find_or_create("baz", false).await?; + assert!(matches!(subdir, Entity::Dir(_))); + + // Find existing entities + let existing_file = dir.find("foo/bar.txt").await?; + assert!(matches!(existing_file, Some(Entity::File(_)))); + + let existing_dir = dir.find("baz").await?; + assert!(matches!(existing_dir, Some(Entity::Dir(_)))); + + Ok(()) + } + + #[tokio::test] + async fn test_ops_list() -> FsResult<()> { + let mut dir = Dir::new(MemoryStore::default()); + + // Create some entries + dir.find_or_create("foo", false).await?; + dir.find_or_create("bar.txt", true).await?; + dir.find_or_create("baz/qux.txt", true).await?; + + // List entries + let entries = dir.list()?; + + assert_eq!(entries.len(), 3); + assert!(entries.contains(&"foo".parse()?)); + assert!(entries.contains(&"bar.txt".parse()?)); + assert!(entries.contains(&"baz".parse()?)); + + Ok(()) + } + + #[tokio::test] + async fn test_ops_copy() -> FsResult<()> { + let mut dir = Dir::new(MemoryStore::default()); + + // Create a source file + assert!(dir.find("source/file.txt").await?.is_none()); + assert!(dir.find("target").await?.is_none()); + dir.find_or_create("source/file.txt", true).await?; + dir.find_or_create("target", false).await?; + + // Copy the file + assert!(dir.find("target/file.txt").await?.is_none()); + dir.copy("source/file.txt", "target").await?; + + // Verify the copy + assert!(matches!( + dir.find("source/file.txt").await?, + Some(Entity::File(_)) + )); + assert!(matches!( + dir.find("target/file.txt").await?, + Some(Entity::File(_)) + )); + + // Test copying a directory + assert!(dir.find("source/subdir").await?.is_none()); + dir.find_or_create("source/subdir", false).await?; + + assert!(dir.find("target/subdir").await?.is_none()); + dir.copy("source/subdir", "target").await?; + + assert!(matches!( + dir.find("source/subdir").await?, + Some(Entity::Dir(_)) + )); + assert!(matches!( + dir.find("target/subdir").await?, + Some(Entity::Dir(_)) + )); + + Ok(()) + } + + #[tokio::test] + async fn test_ops_remove() -> FsResult<()> { + let mut dir = Dir::new(MemoryStore::default()); + + // Create entities to remove + dir.find_or_create("foo/bar.txt", true).await?; + dir.find_or_create("baz", false).await?; + + // Remove file + let (filename, entity) = dir.remove("foo/bar.txt").await?; + assert_eq!(filename, "bar.txt".parse()?); + assert!(matches!(entity, EntityCidLink::Decoded(Entity::File(_)))); + assert!(dir.find("foo/bar.txt").await?.is_none()); + assert!(dir.find("foo").await?.is_some()); + + // Remove directory + let (dirname, entity) = dir.remove("baz").await?; + assert_eq!(dirname, "baz".parse()?); + assert!(matches!(entity, EntityCidLink::Decoded(Entity::Dir(_)))); + assert!(dir.find("baz").await?.is_none()); + + // Try to remove non-existent entity + assert!(dir.remove("nonexistent").await.is_err()); + + Ok(()) + } + + #[tokio::test] + async fn test_ops_complex_nested_hierarchy() -> FsResult<()> { + let mut root = Dir::new(MemoryStore::default()); + + // Create a complex nested structure + root.find_or_create("projects/web/index.html", true).await?; + root.find_or_create("projects/web/styles/main.css", true) + .await?; + root.find_or_create("projects/app/src/main.rs", true) + .await?; + root.find_or_create("documents/personal/notes.txt", true) + .await?; + root.find_or_create("documents/work/report.pdf", true) + .await?; + + // Verify the structure + assert!(matches!(root.find("projects").await?, Some(Entity::Dir(_)))); + assert!(matches!( + root.find("projects/web/index.html").await?, + Some(Entity::File(_)) + )); + assert!(matches!( + root.find("projects/app/src/main.rs").await?, + Some(Entity::File(_)) + )); + assert!(matches!( + root.find("documents/work/report.pdf").await?, + Some(Entity::File(_)) + )); + + // List contents of directories + let root_contents = root.list()?; + assert_eq!(root_contents.len(), 2); + assert!(root_contents.contains(&"projects".parse()?)); + assert!(root_contents.contains(&"documents".parse()?)); + + if let Some(Entity::Dir(projects_dir)) = root.find("projects").await? { + let projects_contents = projects_dir.list()?; + assert_eq!(projects_contents.len(), 2); + assert!(projects_contents.contains(&"web".parse()?)); + assert!(projects_contents.contains(&"app".parse()?)); + } else { + panic!("Expected to find 'projects' directory"); + } + + // Modify a file + if let Some(Entity::File(index_file)) = root.find_mut("projects/web/index.html").await? { + let content = "Hello, World!".as_bytes(); + let content_cid = index_file.get_store().put_raw_block(content).await?; + index_file.set_content(Some(content_cid)); + index_file.store().await?; + } else { + panic!("Expected to find 'index.html' file"); + } + + // Verify the modification + if let Some(Entity::File(index_file)) = root.find("projects/web/index.html").await? { + let content_cid = index_file.get_content().expect("File should have content"); + let content = index_file.get_store().get_raw_block(content_cid).await?; + assert_eq!( + content, + "Hello, World!".as_bytes() + ); + } else { + panic!("Expected to find 'index.html' file"); + } + + // Copy a file + root.copy("documents/personal/notes.txt", "projects") + .await?; + assert!(matches!( + root.find("projects/notes.txt").await?, + Some(Entity::File(_)) + )); + + // Remove a file + let (removed_filename, _) = root.remove("documents/work/report.pdf").await?; + assert_eq!(removed_filename, "report.pdf".parse()?); + assert!(root.find("documents/work/report.pdf").await?.is_none()); + + // Remove a file and its parent directory + root.remove("documents/personal/notes.txt").await?; + root.remove("documents/personal").await?; + assert!(root.find("documents/personal").await?.is_none()); + + // Verify the final structure + assert!(matches!( + root.find("projects/web/index.html").await?, + Some(Entity::File(_)) + )); + assert!(matches!( + root.find("projects/app/src/main.rs").await?, + Some(Entity::File(_)) + )); + assert!(matches!( + root.find("projects/notes.txt").await?, + Some(Entity::File(_)) + )); + assert!(root.find("documents/personal").await?.is_none()); + + Ok(()) + } +} diff --git a/monofs/lib/filesystem/dir/segment.rs b/monofs/lib/filesystem/dir/segment.rs new file mode 100644 index 0000000..e38b908 --- /dev/null +++ b/monofs/lib/filesystem/dir/segment.rs @@ -0,0 +1,291 @@ +use std::{ + fmt::{self, Display}, + str::FromStr, +}; + +use typed_path::{Utf8UnixComponent, Utf8UnixPath, Utf8UnixPathBuf}; + +use crate::filesystem::FsError; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Represents a single segment of a UTF-8 encoded Unix path. +/// +/// This struct provides a way to represent and manipulate individual components +/// of a Unix path, ensuring they are valid UTF-8 strings and non-empty. +/// +/// ## Examples +/// +/// ``` +/// use std::str::FromStr; +/// use monofs::filesystem::Utf8UnixPathSegment; +/// +/// let segment = Utf8UnixPathSegment::from_str("example").unwrap(); +/// +/// assert_eq!(segment.as_str(), "example"); +/// ``` +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Utf8UnixPathSegment(String); + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl Utf8UnixPathSegment { + /// Returns the string representation of the segment. + /// + /// ## Examples + /// + /// ``` + /// use std::str::FromStr; + /// use monofs::filesystem::Utf8UnixPathSegment; + /// + /// let segment = Utf8UnixPathSegment::from_str("example").unwrap(); + /// + /// assert_eq!(segment.as_str(), "example"); + /// ``` + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Returns the bytes representation of the segment. + /// + /// ## Examples + /// + /// ``` + /// use std::str::FromStr; + /// use monofs::filesystem::Utf8UnixPathSegment; + /// + /// let segment = Utf8UnixPathSegment::from_str("example").unwrap(); + /// + /// assert_eq!(segment.as_bytes(), b"example"); + /// ``` + pub fn as_bytes(&self) -> &[u8] { + self.0.as_bytes() + } + + /// Returns the length of the segment in bytes. + /// + /// ## Examples + /// + /// ``` + /// use std::str::FromStr; + /// use monofs::filesystem::Utf8UnixPathSegment; + /// + /// let segment = Utf8UnixPathSegment::from_str("example").unwrap(); + /// + /// assert_eq!(segment.len(), 7); + /// ``` + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns `true` if the segment is empty. + /// + /// ## Examples + /// + /// ``` + /// use std::str::FromStr; + /// use monofs::filesystem::Utf8UnixPathSegment; + /// + /// let segment = Utf8UnixPathSegment::from_str("example").unwrap(); + /// + /// assert_eq!(segment.is_empty(), false); + /// ``` + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl FromStr for Utf8UnixPathSegment { + type Err = FsError; + + fn from_str(s: &str) -> Result { + Utf8UnixPathSegment::try_from(s) + } +} + +impl Display for Utf8UnixPathSegment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl TryFrom<&str> for Utf8UnixPathSegment { + type Error = FsError; + + fn try_from(value: &str) -> Result { + if value.is_empty() { + return Err(FsError::InvalidPathComponent(value.to_string())); + } + + let component = Utf8UnixComponent::try_from(value) + .map_err(|_| FsError::InvalidPathComponent(value.to_string()))?; + + match component { + Utf8UnixComponent::Normal(component) => Ok(Utf8UnixPathSegment(component.to_string())), + _ => Err(FsError::InvalidPathComponent(value.to_string())), + } + } +} + +impl<'a> TryFrom> for Utf8UnixPathSegment { + type Error = FsError; + + fn try_from(component: Utf8UnixComponent<'a>) -> Result { + Utf8UnixPathSegment::try_from(&component) + } +} + +impl<'a> TryFrom<&Utf8UnixComponent<'a>> for Utf8UnixPathSegment { + type Error = FsError; + + fn try_from(component: &Utf8UnixComponent<'a>) -> Result { + match component { + Utf8UnixComponent::Normal(component) => Ok(Utf8UnixPathSegment(component.to_string())), + _ => Err(FsError::InvalidPathComponent(component.to_string())), + } + } +} + +impl<'a> From<&'a Utf8UnixPathSegment> for Utf8UnixComponent<'a> { + fn from(segment: &'a Utf8UnixPathSegment) -> Self { + Utf8UnixComponent::Normal(&segment.0) + } +} + +impl From for Utf8UnixPathBuf { + #[inline] + fn from(segment: Utf8UnixPathSegment) -> Self { + Utf8UnixPathBuf::from(segment.0) + } +} + +impl AsRef<[u8]> for Utf8UnixPathSegment { + #[inline] + fn as_ref(&self) -> &[u8] { + self.as_bytes() + } +} + +impl AsRef for Utf8UnixPathSegment { + #[inline] + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl AsRef for Utf8UnixPathSegment { + #[inline] + fn as_ref(&self) -> &Utf8UnixPath { + Utf8UnixPath::new(self) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_segment_as_str() { + let segment = Utf8UnixPathSegment::from_str("example").unwrap(); + assert_eq!(segment.as_str(), "example"); + } + + #[test] + fn test_segment_as_bytes() { + let segment = Utf8UnixPathSegment::from_str("example").unwrap(); + assert_eq!(segment.as_bytes(), b"example"); + } + + #[test] + fn test_segment_len() { + let segment = Utf8UnixPathSegment::from_str("example").unwrap(); + assert_eq!(segment.len(), 7); + } + + #[test] + fn test_segment_display() { + let segment = Utf8UnixPathSegment::from_str("example").unwrap(); + assert_eq!(format!("{}", segment), "example"); + } + + #[test] + fn test_segment_try_from_str() { + assert!(Utf8UnixPathSegment::try_from("example").is_ok()); + assert!(Utf8UnixPathSegment::from_str("example").is_ok()); + assert!("example".parse::().is_ok()); + + // Negative cases + assert!(Utf8UnixPathSegment::from_str("").is_err()); + assert!(Utf8UnixPathSegment::from_str(".").is_err()); + assert!(Utf8UnixPathSegment::from_str("..").is_err()); + assert!(Utf8UnixPathSegment::from_str("/").is_err()); + assert!(".".parse::().is_err()); + assert!("..".parse::().is_err()); + assert!("/".parse::().is_err()); + assert!("".parse::().is_err()); + assert!("///".parse::().is_err()); + assert!("a/b".parse::().is_err()); + assert!(Utf8UnixPathSegment::try_from(".").is_err()); + assert!(Utf8UnixPathSegment::try_from("..").is_err()); + assert!(Utf8UnixPathSegment::try_from("/").is_err()); + assert!(Utf8UnixPathSegment::try_from("").is_err()); + assert!(Utf8UnixPathSegment::try_from("///").is_err()); + assert!(Utf8UnixPathSegment::try_from("a/b").is_err()); + } + + #[test] + fn test_segment_utf8_characters() { + assert!(Utf8UnixPathSegment::try_from("файл").is_ok()); + assert!(Utf8UnixPathSegment::try_from("文件").is_ok()); + assert!(Utf8UnixPathSegment::try_from("🚀").is_ok()); + + // Negative cases + assert!(Utf8UnixPathSegment::try_from("файл/имя").is_err()); + assert!(Utf8UnixPathSegment::try_from("文件/名称").is_err()); + } + + #[test] + fn test_segment_from_utf8_unix_path_segment_to_utf8_unix_component() { + let segment = Utf8UnixPathSegment::from_str("example").unwrap(); + assert_eq!( + Utf8UnixComponent::from(&segment), + Utf8UnixComponent::Normal("example") + ); + } + + #[test] + fn test_segment_from_utf8_unix_path_segment_to_utf8_unix_path_buf() { + let segment = Utf8UnixPathSegment::from_str("example").unwrap(); + assert_eq!( + Utf8UnixPathBuf::from(segment), + Utf8UnixPathBuf::from("example") + ); + } + + #[test] + fn test_segment_normal_with_special_characters() { + assert!(Utf8UnixPathSegment::try_from("file.txt").is_ok()); + assert!(Utf8UnixPathSegment::try_from("file-name").is_ok()); + assert!(Utf8UnixPathSegment::try_from("file_name").is_ok()); + assert!(Utf8UnixPathSegment::try_from("file name").is_ok()); + assert!(Utf8UnixPathSegment::try_from("file\\name").is_ok()); + assert!(Utf8UnixPathSegment::try_from("file:name").is_ok()); + assert!(Utf8UnixPathSegment::try_from("file*name").is_ok()); + assert!(Utf8UnixPathSegment::try_from("file?name").is_ok()); + + // Negative cases + assert!(Utf8UnixPathSegment::try_from("file/name").is_err()); + } +} diff --git a/monofs/lib/filesystem/entity.rs b/monofs/lib/filesystem/entity.rs new file mode 100644 index 0000000..cb739b1 --- /dev/null +++ b/monofs/lib/filesystem/entity.rs @@ -0,0 +1,138 @@ +use std::fmt::{self, Debug}; + +use monoutils_store::{ipld::cid::Cid, IpldStore, Storable, StoreError, StoreResult}; + +use crate::filesystem::{Dir, File, FsError, FsResult, Metadata, SoftLink}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// This is an entity in the file system. +#[derive(Clone)] +pub enum Entity +where + S: IpldStore, +{ + /// A file. + File(File), + + /// A directory. + Dir(Dir), + + /// A softlink. + SoftLink(SoftLink), +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl Entity +where + S: IpldStore, +{ + /// Returns true if the entity is a file. + pub fn is_file(&self) -> bool { + matches!(self, Entity::File(_)) + } + + /// Returns true if the entity is a directory. + pub fn is_dir(&self) -> bool { + matches!(self, Entity::Dir(_)) + } + + /// Tries to convert the entity to a file. + pub fn into_file(self) -> FsResult> { + if let Entity::File(file) = self { + return Ok(file); + } + Err(FsError::NotAFile(String::new())) + } + + /// Tries to convert the entity to a directory. + pub fn into_dir(self) -> FsResult> { + if let Entity::Dir(dir) = self { + return Ok(dir); + } + + Err(FsError::NotADirectory(String::new())) + } + + /// Tries to convert the entity to a softlink. + pub fn into_softlink(self) -> FsResult> { + if let Entity::SoftLink(softlink) = self { + return Ok(softlink); + } + + Err(FsError::NotASoftLink(String::new())) + } + + /// Returns the metadata for the directory. + pub fn get_metadata(&self) -> &Metadata { + match self { + Entity::File(file) => file.get_metadata(), + Entity::Dir(dir) => dir.get_metadata(), + Entity::SoftLink(softlink) => softlink.get_metadata(), + } + } + + /// Returns the store used to persist the entity. + pub fn get_store(&self) -> &S { + match self { + Entity::File(file) => file.get_store(), + Entity::Dir(dir) => dir.get_store(), + Entity::SoftLink(softlink) => softlink.get_store(), + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Storable for Entity +where + S: IpldStore + Send + Sync, +{ + async fn store(&self) -> StoreResult { + match self { + Entity::File(file) => file.store().await, + Entity::Dir(dir) => dir.store().await, + Entity::SoftLink(softlink) => softlink.store().await, + } + } + + async fn load(cid: &Cid, store: S) -> StoreResult { + // The order of the following `if let` statements is important because for some reason + // Directory entity deserializes successfully into a File entity even though they have + // different structure. This is likely due to the way `serde_ipld_dagcbor` deserializes + // the entities. + if let Ok(softlink) = SoftLink::load(cid, store.clone()).await { + return Ok(Entity::SoftLink(softlink)); + } + + if let Ok(dir) = Dir::load(cid, store.clone()).await { + return Ok(Entity::Dir(dir)); + } + + if let Ok(file) = File::load(cid, store.clone()).await { + return Ok(Entity::File(file)); + } + + Err(StoreError::custom(FsError::UnableToLoadEntity(*cid))) + } +} + +impl Debug for Entity +where + S: IpldStore, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Entity::File(file) => f.debug_tuple("File").field(file).finish(), + Entity::Dir(dir) => f.debug_tuple("Dir").field(dir).finish(), + Entity::SoftLink(softlink) => f.debug_tuple("SoftLink").field(softlink).finish(), + } + } +} diff --git a/monofs/lib/filesystem/eq.rs b/monofs/lib/filesystem/eq.rs new file mode 100644 index 0000000..8f242ff --- /dev/null +++ b/monofs/lib/filesystem/eq.rs @@ -0,0 +1,67 @@ +use futures::future::BoxFuture; + +//-------------------------------------------------------------------------------------------------- +// Traits +//-------------------------------------------------------------------------------------------------- + +/// An async version of the `PartialEq` trait. +pub trait AsyncPartialEq +where + Self: Send + Sync, +{ + /// This method tests for `self` and `other` values to be equal, and returns a future that + /// resolves to `true` if they are equal. + fn eq<'a>(&'a self, other: &'a Rhs) -> BoxFuture<'a, bool>; + + /// This method tests for `!=` and returns a future that resolves to `true` if the values are not equal. + fn ne<'a>(&'a self, other: &'a Rhs) -> BoxFuture<'a, bool> + where + Self: 'a, + { + Box::pin(async move { !self.eq(other).await }) + } +} + +/// An async version of the `Eq` trait. +pub trait AsyncEq: AsyncPartialEq {} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl AsyncPartialEq for T +where + T: PartialEq + Send + Sync, +{ + fn eq<'a>(&'a self, other: &'a T) -> BoxFuture<'a, bool> { + Box::pin(async move { self == other }) + } +} + +//-------------------------------------------------------------------------------------------------- +// Macros +//-------------------------------------------------------------------------------------------------- + +/// ... +#[macro_export] +macro_rules! async_assert { + ($lhs:expr, $rhs:expr) => { + assert!($lhs.await) + }; +} + +/// ... +#[macro_export] +macro_rules! async_assert_eq { + ($lhs:expr, $rhs:expr) => { + assert!($lhs.eq($rhs).await) + }; +} + +/// ... +#[macro_export] +macro_rules! async_assert_ne { + ($lhs:expr, $rhs:expr) => { + assert!($lhs.ne($rhs).await) + }; +} diff --git a/monofs/lib/filesystem/error.rs b/monofs/lib/filesystem/error.rs new file mode 100644 index 0000000..d3ba866 --- /dev/null +++ b/monofs/lib/filesystem/error.rs @@ -0,0 +1,166 @@ +use std::{error::Error, fmt::Display}; + +use monoutils_store::ipld::cid::Cid; +use thiserror::Error; + +use crate::filesystem::Utf8UnixPathSegment; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// The result of a file system operation. +pub type FsResult = Result; + +/// An error that occurred during a file system operation. +#[derive(Debug, Error)] +pub enum FsError { + /// Infallible error. + #[error("Infallible error")] + Infallible(#[from] core::convert::Infallible), + + /// Not a file. + #[error("Not a file: {0:?}")] + NotAFile(String), + + /// Not a directory. + #[error("Not a directory: {0:?}")] + NotADirectory(String), + + /// Not a softlink. + #[error("Not a softlink: {0:?}")] + NotASoftLink(String), + + /// Path not found. + #[error("Path not found: {0}")] + PathNotFound(String), + + // /// UCAN error. + // #[error("UCAN error: {0}")] + // Ucan(#[from] monoutils_ucan::UcanError), + /// Custom error. + #[error("Custom error: {0}")] + Custom(#[from] AnyError), + + // /// DID related error. + // #[error("DID error: {0}")] + // Did(#[from] monoutils_did_wk::DidError), + /// IPLD Store error. + #[error("IPLD Store error: {0}")] + IpldStore(#[from] monoutils_store::StoreError), + + /// Invalid deserialized OpenFlag value + #[error("Invalid OpenFlag value: {0}")] + InvalidOpenFlag(u8), + + /// Invalid deserialized EntityFlag value + #[error("Invalid EntityFlag value: {0}")] + InvalidEntityFlag(u8), + + /// Invalid deserialized PathFlag value + #[error("Invalid PathFlag value: {0}")] + InvalidPathFlag(u8), + + /// Invalid path component + #[error("Invalid path component: {0}")] + InvalidPathComponent(String), + + /// Invalid search path with root. + #[error("Invalid search path: {0}")] + InvalidSearchPath(String), + + /// SoftLink not supported yet. + #[error("SoftLink not supported yet: path: {0:?}")] + SoftLinkNotSupportedYet(Vec), + + /// Invalid search path empty. + #[error("Invalid search path empty")] + InvalidSearchPathEmpty, + + /// Unable to load entity. + #[error("Unable to load entity: {0}")] + UnableToLoadEntity(Cid), + + /// CID error. + #[error("CID error: {0}")] + CidError(#[from] monoutils_store::ipld::cid::Error), + + /// Path has root. + #[error("Path has root: {0}")] + PathHasRoot(String), + + /// Source is not a directory. + #[error("Source is not a directory: {0}")] + SourceIsNotADir(String), + + /// Target is not a directory. + #[error("Target is not a directory: {0}")] + TargetIsNotADir(String), + + /// Path is empty. + #[error("Path is empty")] + PathIsEmpty, + + /// Maximum follow depth reached. + #[error("Maximum follow depth reached")] + MaxFollowDepthReached, + + /// Broken softlink. + #[error("Broken softlink: {0}")] + BrokenSoftLink(Cid), +} + +// /// Permission error. +// #[derive(Debug, Error)] +// pub enum PermissionError { +// /// Child descriptor has higher permission than parent. +// #[error("Child descriptor has higher permission than parent: path: {0}, parent(descriptor_flags: {1:?}) child (descriptor_flags: {2:?}, open_flags: {3:?})")] +// ChildPermissionEscalation(Path, DescriptorFlags, DescriptorFlags, OpenFlags), +// } + +/// An error that can represent any error. +#[derive(Debug)] +pub struct AnyError { + error: anyhow::Error, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl FsError { + /// Creates a new `Err` result. + pub fn custom(error: impl Into) -> FsError { + FsError::Custom(AnyError { + error: error.into(), + }) + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Creates an `Ok` `FsResult`. +#[allow(non_snake_case)] +pub fn Ok(value: T) -> FsResult { + Result::Ok(value) +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl PartialEq for AnyError { + fn eq(&self, other: &Self) -> bool { + self.error.to_string() == other.error.to_string() + } +} + +impl Display for AnyError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.error) + } +} + +impl Error for AnyError {} diff --git a/monofs/lib/filesystem/file/file.rs b/monofs/lib/filesystem/file/file.rs new file mode 100644 index 0000000..bf8ca00 --- /dev/null +++ b/monofs/lib/filesystem/file/file.rs @@ -0,0 +1,428 @@ +use std::{ + fmt::{self, Debug}, + sync::Arc, +}; + +use bytes::Bytes; +use monoutils_store::{ + ipld::cid::Cid, IpldReferences, IpldStore, Storable, StoreError, StoreResult, +}; +use serde::{ + de::{self, DeserializeSeed}, + Deserialize, Deserializer, Serialize, +}; + +use crate::filesystem::{kind::EntityType, FsError, FsResult, Metadata}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Represents a file node in the `monofs` _immutable_ file system. +/// +/// ## Important +/// +/// Entities in `monofs` are designed to be immutable and clone-on-write meaning writes create +/// forks of the entity. +#[derive(Clone)] +pub struct File +where + S: IpldStore, +{ + inner: Arc>, +} + +#[derive(Clone)] +struct FileInner +where + S: IpldStore, +{ + /// File metadata. + pub(crate) metadata: Metadata, + + /// File content. If the file is empty, this will be `None`. + pub(crate) content: Option, + + /// The store used to persist blocks in the file. + pub(crate) store: S, +} + +//-------------------------------------------------------------------------------------------------- +// Types: Serializable +//-------------------------------------------------------------------------------------------------- + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct FileSerializable { + metadata: Metadata, + content: Option, +} + +pub(crate) struct FileDeserializeSeed { + pub(crate) store: S, +} + +//-------------------------------------------------------------------------------------------------- +// Methods: File +//-------------------------------------------------------------------------------------------------- + +impl File +where + S: IpldStore, +{ + /// Creates a new empty file. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::File; + /// use monoutils_store::MemoryStore; + /// + /// let store = MemoryStore::default(); + /// let file = File::new(store); + /// + /// assert!(file.is_empty()); + /// ``` + pub fn new(store: S) -> Self { + Self { + inner: Arc::new(FileInner { + metadata: Metadata::new(EntityType::File), + content: None, + store, + }), + } + } + + /// Creates a new file with the given content. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::File; + /// use monoutils_store::MemoryStore; + /// + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// let store = MemoryStore::default(); + /// let file = File::with_content(store, b"Hello, World!".to_vec()).await; + /// + /// assert!(!file.is_empty()); + /// assert!(file.get_content().is_some()); + /// # Ok(()) + /// # } + /// ``` + pub async fn with_content(store: S, content: impl Into + Send) -> Self { + let cid = store.put_raw_block(content).await.unwrap(); + + Self { + inner: Arc::new(FileInner { + metadata: Metadata::new(EntityType::File), + content: Some(cid), + store, + }), + } + } + + /// Returns the content of the file. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::File; + /// use monoutils_store::MemoryStore; + /// + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// let store = MemoryStore::default(); + /// let file = File::with_content(store, b"Hello, World!".to_vec()).await; + /// + /// assert!(file.get_content().is_some()); + /// # Ok(()) + /// # } + /// ``` + pub fn get_content(&self) -> Option<&Cid> { + self.inner.content.as_ref() + } + + /// Sets the content of the file. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::File; + /// use monoutils_store::{MemoryStore, ipld::cid::Cid}; + /// + /// let store = MemoryStore::default(); + /// let mut file = File::new(store); + /// + /// let content_cid = Cid::default(); + /// file.set_content(Some(content_cid)); + /// + /// assert!(!file.is_empty()); + /// assert_eq!(file.get_content(), Some(&content_cid)); + /// ``` + pub fn set_content(&mut self, content: Option) { + let inner = Arc::make_mut(&mut self.inner); + inner.content = content; + } + + /// Returns the metadata for the file. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::{File, EntityType}; + /// use monoutils_store::MemoryStore; + /// + /// let store = MemoryStore::default(); + /// let file = File::new(store); + /// + /// assert_eq!(file.get_metadata().get_entity_type(), &EntityType::File); + /// ``` + pub fn get_metadata(&self) -> &Metadata { + &self.inner.metadata + } + + /// Returns the store used to persist the file. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::File; + /// use monoutils_store::{MemoryStore, IpldStore}; + /// + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// let store = MemoryStore::default(); + /// let file = File::new(store); + /// + /// assert!(file.get_store().is_empty().await?); + /// # Ok(()) + /// # } + /// ``` + pub fn get_store(&self) -> &S { + &self.inner.store + } + + /// Returns `true` if the file is empty. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::File; + /// use monoutils_store::MemoryStore; + /// + /// let store = MemoryStore::default(); + /// let file = File::new(store); + /// + /// assert!(file.is_empty()); + /// ``` + pub fn is_empty(&self) -> bool { + self.inner.content.is_none() + } + + /// Truncates the file to zero bytes. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::File; + /// use monoutils_store::{MemoryStore, ipld::cid::Cid}; + /// + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// let store = MemoryStore::default(); + /// let mut file = File::with_content(store, b"Hello, World!".to_vec()).await; + /// + /// assert!(!file.is_empty()); + /// + /// file.truncate(); + /// + /// assert!(file.is_empty()); + /// assert!(file.get_content().is_none()); + /// # Ok(()) + /// # } + /// ``` + pub fn truncate(&mut self) { + let inner = Arc::make_mut(&mut self.inner); + inner.content = None; + } + + /// Deserializes to a `Dir` using an arbitrary deserializer and store. + pub fn deserialize_with<'de>( + deserializer: impl Deserializer<'de, Error: Into>, + store: S, + ) -> FsResult { + FileDeserializeSeed::new(store) + .deserialize(deserializer) + .map_err(Into::into) + } + + /// Tries to create a new `Dir` from a serializable representation. + pub(crate) fn try_from_serializable( + serializable: FileSerializable, + store: S, + ) -> FsResult { + Ok(File { + inner: Arc::new(FileInner { + metadata: serializable.metadata, + content: serializable.content, + store, + }), + }) + } +} + +//-------------------------------------------------------------------------------------------------- +// Methods: FileDeserializeSeed +//-------------------------------------------------------------------------------------------------- + +impl FileDeserializeSeed { + fn new(store: S) -> Self { + Self { store } + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations: File +//-------------------------------------------------------------------------------------------------- + +impl Storable for File +where + S: IpldStore + Send + Sync, +{ + async fn store(&self) -> StoreResult { + let serializable = FileSerializable { + metadata: self.inner.metadata.clone(), + content: self.inner.content, + }; + + self.inner.store.put_node(&serializable).await + } + + async fn load(cid: &Cid, store: S) -> StoreResult { + let serializable = store.get_node(cid).await?; + File::try_from_serializable(serializable, store).map_err(StoreError::custom) + } +} + +impl Debug for File +where + S: IpldStore, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("File") + .field("metadata", &self.inner.metadata) + .field("content", &self.inner.content) + .finish() + } +} + +impl PartialEq for File +where + S: IpldStore, +{ + fn eq(&self, other: &Self) -> bool { + self.inner.metadata == other.inner.metadata && self.inner.content == other.inner.content + } +} + +impl IpldReferences for FileSerializable { + fn get_references<'a>(&'a self) -> Box + Send + 'a> { + match self.content.as_ref() { + Some(cid) => Box::new(std::iter::once(cid)), + None => Box::new(std::iter::empty()), + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations: FileDeserializeSeed +//-------------------------------------------------------------------------------------------------- + +impl<'de, S> DeserializeSeed<'de> for FileDeserializeSeed +where + S: IpldStore, +{ + type Value = File; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let serializable = FileSerializable::deserialize(deserializer)?; + File::try_from_serializable(serializable, self.store).map_err(de::Error::custom) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use monoutils_store::MemoryStore; + + use super::*; + + #[test] + fn test_file_new() { + let file = File::new(MemoryStore::default()); + + assert!(file.is_empty()); + assert_eq!(file.get_metadata().get_entity_type(), &EntityType::File); + assert!(file.get_content().is_none()); + } + + #[tokio::test] + async fn test_file_with_content() -> anyhow::Result<()> { + let store = MemoryStore::default(); + let file = File::with_content(store.clone(), b"Hello, World!".to_vec()).await; + assert!(!file.is_empty()); + + let content_cid = file.get_content().unwrap(); + assert_eq!( + store.get_raw_block(content_cid).await?, + Bytes::from(b"Hello, World!".to_vec()) + ); + + Ok(()) + } + + #[test] + fn test_file_set_content() { + let mut file = File::new(MemoryStore::default()); + + let content_cid = Cid::default(); + file.set_content(Some(content_cid)); + + assert!(!file.is_empty()); + assert_eq!(file.get_content(), Some(&content_cid)); + } + + #[test] + fn test_file_truncate() { + let mut file = File::new(MemoryStore::default()); + + let content_cid = Cid::default(); + file.set_content(Some(content_cid)); + assert!(!file.is_empty()); + + file.truncate(); + assert!(file.is_empty()); + assert!(file.get_content().is_none()); + } + + #[tokio::test] + async fn test_file_store_and_load() { + let store = MemoryStore::default(); + let mut file = File::new(store.clone()); + + let content_cid = Cid::default(); + file.set_content(Some(content_cid)); + + let stored_cid = file.store().await.unwrap(); + let loaded_file = File::load(&stored_cid, store).await.unwrap(); + + assert_eq!(file, loaded_file); + } +} diff --git a/monofs/lib/filesystem/file/io.rs b/monofs/lib/filesystem/file/io.rs new file mode 100644 index 0000000..2bbd931 --- /dev/null +++ b/monofs/lib/filesystem/file/io.rs @@ -0,0 +1,180 @@ +use std::{ + io, + pin::Pin, + task::{Context, Poll}, +}; + +use futures::Future; +use monoutils_store::IpldStore; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +use crate::filesystem::{File, FsResult}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A stream for reading from a `File` asynchronously. +pub struct FileInputStream<'a> { + reader: Pin>, +} + +/// A stream for writing to a `File` asynchronously. +pub struct FileOutputStream<'a, S> +where + S: IpldStore, +{ + file: &'a mut File, + buffer: Vec, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl<'a> FileInputStream<'a> { + /// Creates a new `FileInputStream` from a `File`. + pub async fn new(file: &'a File) -> io::Result + where + S: IpldStore + Send + Sync + 'static, + { + let store = file.get_store(); + let reader = match file.get_content() { + Some(cid) => store + .get_bytes(cid) + .await + .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?, + None => Box::pin(tokio::io::empty()) as Pin>, + }; + + Ok(Self { reader }) + } +} + +impl<'a, S> FileOutputStream<'a, S> +where + S: IpldStore + Send + Sync + 'static, +{ + /// Creates a new `FileOutputStream` for a `File`. + pub fn new(file: &'a mut File) -> Self { + Self { + file, + buffer: Vec::new(), + } + } + + /// Finalizes the write operation and updates the file content. + async fn finalize(&mut self) -> FsResult<()> { + if !self.buffer.is_empty() { + let store = self.file.get_store(); + let cid = store.put_bytes(&self.buffer[..]).await.map(Into::into)?; + self.file.set_content(Some(cid)); + self.buffer.clear(); + } + Ok(()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl AsyncRead for FileInputStream<'_> { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + self.reader.as_mut().poll_read(cx, buf) + } +} + +impl AsyncWrite for FileOutputStream<'_, S> +where + S: IpldStore + Send + Sync + 'static, +{ + fn poll_write( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + self.buffer.extend_from_slice(buf); + Poll::Ready(Ok(buf.len())) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + let finalize_future = self.finalize(); + tokio::pin!(finalize_future); + + finalize_future + .poll(cx) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use anyhow::Result; + use monoutils_store::MemoryStore; + use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; + + use crate::filesystem::File; + + use super::*; + + #[tokio::test] + async fn test_file_input_stream() -> Result<()> { + let store = MemoryStore::default(); + let mut file = File::new(store.clone()); + + // Create some content for the file + let content = b"Hello, world!"; + let cid = store.put_bytes(content.as_slice()).await?; + file.set_content(Some(cid)); + + // Create an input stream from the file + let mut input_stream = FileInputStream::new(&file).await?; + + // Read the content from the input stream + let mut buffer = Vec::new(); + let n = input_stream.read_to_end(&mut buffer).await?; + + // Verify the content + assert_eq!(n, content.len()); + assert_eq!(buffer, content); + + Ok(()) + } + + #[tokio::test] + async fn test_file_output_stream() -> Result<()> { + let store = MemoryStore::default(); + let mut file = File::new(store); + let mut output_stream = FileOutputStream::new(&mut file); + + let data = b"Hello, world!"; + output_stream.write_all(data).await?; + output_stream.shutdown().await?; + + // Now read the file to verify the content + let input_stream = FileInputStream::new(&file).await?; + let mut buf = BufReader::new(input_stream); + let mut content = Vec::new(); + buf.read_to_end(&mut content).await?; + + assert_eq!(content, data); + + Ok(()) + } +} diff --git a/monofs/lib/filesystem/file/mod.rs b/monofs/lib/filesystem/file/mod.rs new file mode 100644 index 0000000..c95e636 --- /dev/null +++ b/monofs/lib/filesystem/file/mod.rs @@ -0,0 +1,11 @@ +//! File implementation. + +mod file; +mod io; + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +pub use file::*; +pub use io::*; diff --git a/monofs/lib/filesystem/kind.rs b/monofs/lib/filesystem/kind.rs new file mode 100644 index 0000000..1d7f382 --- /dev/null +++ b/monofs/lib/filesystem/kind.rs @@ -0,0 +1,21 @@ +use serde::{Deserialize, Serialize}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// The kind of an entity in the file system. +/// +/// This corresponds to `descriptor-type` in the WASI. `monofs` does not support all the types that WASI +/// supports. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum EntityType { + /// The entity is a regular file. + File, + + /// The entity is a directory. + Dir, + + /// The entity is a symbolic link. + SoftLink, +} diff --git a/monofs/lib/filesystem/link/cidlink/attributes.rs b/monofs/lib/filesystem/link/cidlink/attributes.rs new file mode 100644 index 0000000..1adfc8c --- /dev/null +++ b/monofs/lib/filesystem/link/cidlink/attributes.rs @@ -0,0 +1,12 @@ +use crate::filesystem::{CidLink, ExtendedAttributes}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A link representing an association between [`Cid`] and a lazily loaded [`ExtendedAttributes`]. +pub type AttributeCidLink = CidLink>; + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- diff --git a/monofs/lib/filesystem/link/cidlink/entity.rs b/monofs/lib/filesystem/link/cidlink/entity.rs new file mode 100644 index 0000000..1ab00ea --- /dev/null +++ b/monofs/lib/filesystem/link/cidlink/entity.rs @@ -0,0 +1,102 @@ +use monoutils_store::IpldStore; + +use crate::filesystem::{ + dir::Dir, entity::Entity, file::File, softlink::SoftLink, CidLink, FsResult, +}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A link representing an association between [`Cid`] and a lazily loaded [`Entity`] or just the +/// [`Entity`] itself. +pub type EntityCidLink = CidLink>; + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl EntityCidLink +where + S: IpldStore, +{ + /// Gets the [`Entity`] that this link points to. + /// + /// This will not resolve the [`Entity`] from the store if it is not already fetched and + /// decoded. + #[inline] + pub fn get_entity(&self) -> Option<&Entity> { + self.get_value() + } + + /// Gets a mutable reference to the [`Entity`] that this link points to. + /// + /// This will not resolve the [`Entity`] from the store if it is not already fetched and + /// decoded. + #[inline] + pub fn get_entity_mut(&mut self) -> Option<&mut Entity> { + self.get_value_mut() + } + + /// Resolves the [`Entity`] that this link points to. + /// + /// This will attempt to resolve the [`Entity`] from the store if it is not already decoded. + #[inline] + pub async fn resolve_entity(&self, store: S) -> FsResult<&Entity> + where + S: Send + Sync, + { + self.resolve_value(store).await + } + + /// Resolves the [`Entity`] that this link points to. + /// + /// This will attempt to resolve the [`Entity`] from the store if it is not already decoded. + #[inline] + pub async fn resolve_entity_mut(&mut self, store: S) -> FsResult<&mut Entity> + where + S: Send + Sync, + { + self.resolve_value_mut(store).await + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl From> for EntityCidLink +where + S: IpldStore, +{ + fn from(entity: Entity) -> Self { + Self::Decoded(entity) + } +} + +impl From> for EntityCidLink +where + S: IpldStore, +{ + fn from(dir: Dir) -> Self { + Self::Decoded(Entity::Dir(dir)) + } +} + +impl From> for EntityCidLink +where + S: IpldStore, +{ + fn from(file: File) -> Self { + Self::Decoded(Entity::File(file)) + } +} + +impl From> for EntityCidLink +where + S: IpldStore, +{ + fn from(softlink: SoftLink) -> Self { + Self::Decoded(Entity::SoftLink(softlink)) + } +} diff --git a/monofs/lib/filesystem/link/cidlink/mod.rs b/monofs/lib/filesystem/link/cidlink/mod.rs new file mode 100644 index 0000000..f7e6e90 --- /dev/null +++ b/monofs/lib/filesystem/link/cidlink/mod.rs @@ -0,0 +1,9 @@ +mod attributes; +mod entity; + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +pub use attributes::*; +pub use entity::*; diff --git a/monofs/lib/filesystem/link/link.rs b/monofs/lib/filesystem/link/link.rs new file mode 100644 index 0000000..4e3895a --- /dev/null +++ b/monofs/lib/filesystem/link/link.rs @@ -0,0 +1,195 @@ +use std::{ + fmt::{self, Display}, + str::FromStr, +}; + +use async_once_cell::OnceCell; +use async_recursion::async_recursion; +use monoutils_store::{ipld::cid::Cid, IpldStore, Storable}; + +use crate::filesystem::{FsError, FsResult}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A type alias for `OnceCell` holding a lazily initialized value. +pub type Cached = OnceCell; + +/// A link representing an association between an identifier and some lazily loaded value or +/// just the value itself. +pub enum Link { + /// A link that is encoded and needs to be resolved. + Encoded { + /// The identifier of the link, e.g. a URI or CID. + identifier: I, + + /// The cached value associated with the identifier. + cached: Cached, + }, + + /// A link that is decoded and can be used directly. + Decoded(V), +} + +/// A link representing an association between [`Cid`] and some lazily loaded value. +pub type CidLink = Link; + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl CidLink { + /// Gets the value that this link points to. + pub fn get_value(&self) -> Option<&V> { + match self { + Self::Encoded { cached, .. } => cached.get(), + Self::Decoded(value) => Some(value), + } + } + + /// Gets a mutable reference to the value that this link points to. + pub fn get_value_mut(&mut self) -> Option<&mut V> { + match self { + Self::Encoded { cached, .. } => cached.get_mut(), + Self::Decoded(value) => Some(value), + } + } + + /// Gets the [`Cid`] of the [`Entity`] that this link points to. + /// + /// This will not encode the [`Cid`] if it is not already encoded. + pub fn get_cid(&self) -> Option<&Cid> { + match self { + Self::Encoded { identifier, .. } => Some(identifier), + Self::Decoded(_) => None, + } + } + + /// Resolves the [`Entity`]'s [`Cid`]. + /// + /// This will attempt to encode the [`Entity`] if it is not already encoded. + #[async_recursion(?Send)] + pub async fn resolve_cid(&self) -> FsResult + where + S: IpldStore, + V: Storable, + { + match self { + Self::Encoded { identifier, .. } => Ok(*identifier), + Self::Decoded(value) => Ok(value.store().await?), + } + } + + /// Resolves the value that this link points to. + /// + /// This will attempt to resolve the value from the store if it is not already decoded. + pub async fn resolve_value(&self, store: S) -> FsResult<&V> + where + S: IpldStore + Send + Sync, + V: Storable, + { + match self { + Self::Encoded { identifier, cached } => cached + .get_or_try_init(V::load(identifier, store)) + .await + .map_err(Into::into), + Self::Decoded(value) => Ok(value), + } + } + + /// Resolves the value that this link points to. + /// + /// This will attempt to resolve the value from the store if it is not already decoded. + pub async fn resolve_value_mut(&mut self, store: S) -> FsResult<&mut V> + where + S: IpldStore + Send + Sync, + V: Storable, + { + match self { + Self::Encoded { identifier, cached } => { + cached.get_or_try_init(V::load(identifier, store)).await?; + Ok(cached.get_mut().unwrap()) + } + Self::Decoded(value) => Ok(value), + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Clone for Link +where + I: Clone, + V: Clone, +{ + fn clone(&self) -> Self { + match self { + Self::Encoded { identifier, .. } => Self::Encoded { + identifier: identifier.clone(), + cached: Cached::new(), + }, + Self::Decoded(value) => Self::Decoded(value.clone()), + } + } +} + +impl From for CidLink { + fn from(cid: Cid) -> Self { + Self::Encoded { + identifier: cid, + cached: Cached::new(), + } + } +} + +impl FromStr for CidLink { + type Err = FsError; + + fn from_str(s: &str) -> Result { + let cid = Cid::from_str(s)?; + Ok(Self::from(cid)) + } +} + +impl TryFrom for CidLink { + type Error = FsError; + + fn try_from(value: String) -> Result { + let cid = Cid::try_from(value)?; + Ok(Self::from(cid)) + } +} + +impl Display for CidLink +where + T: Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Encoded { identifier, .. } => write!(f, "{}", identifier), + Self::Decoded(value) => write!(f, "{}", value), + } + } +} + +impl fmt::Debug for CidLink +where + T: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Encoded { identifier, cached } => f + .debug_struct("CidLink") + .field("identifier", &identifier) + .field("cached", &cached.get()) + .finish(), + Self::Decoded(value) => f + .debug_struct("CidLink") + .field("identifier", &value) + .finish(), + } + } +} diff --git a/monofs/lib/filesystem/link/mod.rs b/monofs/lib/filesystem/link/mod.rs new file mode 100644 index 0000000..d422273 --- /dev/null +++ b/monofs/lib/filesystem/link/mod.rs @@ -0,0 +1,9 @@ +mod cidlink; +mod link; + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +pub use cidlink::*; +pub use link::*; diff --git a/monofs/lib/filesystem/metadata.rs b/monofs/lib/filesystem/metadata.rs new file mode 100644 index 0000000..d20ac93 --- /dev/null +++ b/monofs/lib/filesystem/metadata.rs @@ -0,0 +1,153 @@ +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use getset::Getters; +use serde::{Deserialize, Serialize}; + +use crate::config::DEFAULT_SOFTLINK_DEPTH; + +use super::kind::EntityType; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Relevant metadata for a file system entity. +/// +/// This mostly corresponds to the `fd-stat` in POSIX. `monofs` does not support +/// hard links, so there is no `link-count` field. Also `size` is not stored here, but rather +/// requested when needed. +/// +/// ## Examples +/// +/// ``` +/// use monofs::filesystem::{EntityType, Metadata, SyncType}; +/// use monofs::config::DEFAULT_SOFTLINK_DEPTH; +/// +/// let metadata = Metadata::new(EntityType::File); +/// assert_eq!(*metadata.get_entity_type(), EntityType::File); +/// assert_eq!(*metadata.get_sync_type(), SyncType::RAFT); +/// assert_eq!(*metadata.get_softlink_depth(), DEFAULT_SOFTLINK_DEPTH); +/// ``` +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Getters)] +#[getset(get = "pub with_prefix")] +pub struct Metadata { + /// The type of the entity. + entity_type: EntityType, + + /// The time the entity was created. + created_at: DateTime, + + /// The time of the last modification of the entity. + modified_at: DateTime, + + /// The size of the entity in bytes. + sync_type: SyncType, + + /// The maximum depth of a softlink. + softlink_depth: u32, + // /// Extended attributes. + // #[serde(skip)] + // extended_attrs: Option>, +} + +/// The method of syncing to use for the entity used by the filesystem +/// service. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum SyncType { + /// Use the [RAFT consensus algorithm][raft] to sync the entity. + /// + /// [raft]: https://raft.github.io/ + #[default] + RAFT, + + /// Use [Merkle-CRDT][crdt] as the method of syncing. + /// + /// [crdt]: https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type + CRDT, +} + +/// Extended attributes for a file system entity. +pub struct ExtendedAttributes { + /// The map of extended attributes. + _map: BTreeMap, + + /// The store used to persist the extended attributes. + _store: S, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl Metadata { + /// Creates a new metadata object. + /// + /// ## Examples + /// + /// ``` + /// use monofs::filesystem::{EntityType, Metadata, SyncType}; + /// use monofs::config::DEFAULT_SOFTLINK_DEPTH; + /// + /// let metadata = Metadata::new(EntityType::File); + /// assert_eq!(*metadata.get_entity_type(), EntityType::File); + /// assert_eq!(*metadata.get_sync_type(), SyncType::RAFT); + /// assert_eq!(*metadata.get_softlink_depth(), DEFAULT_SOFTLINK_DEPTH); + /// ``` + pub fn new(entity_type: EntityType) -> Self { + let now = Utc::now(); + + Self { + entity_type, + created_at: now, + modified_at: now, + sync_type: SyncType::default(), + softlink_depth: DEFAULT_SOFTLINK_DEPTH, + // extended_attrs: None, + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_metadata_new() { + let metadata = Metadata::new(EntityType::File); + + assert_eq!(*metadata.get_entity_type(), EntityType::File); + assert_eq!(*metadata.get_sync_type(), SyncType::RAFT); + assert_eq!(*metadata.get_softlink_depth(), DEFAULT_SOFTLINK_DEPTH); + } + + #[test] + fn test_metadata_getters() { + let metadata = Metadata::new(EntityType::Dir); + + assert_eq!(*metadata.get_entity_type(), EntityType::Dir); + assert_eq!(*metadata.get_sync_type(), SyncType::RAFT); + assert_eq!(*metadata.get_softlink_depth(), DEFAULT_SOFTLINK_DEPTH); + assert!(metadata.get_created_at() <= &Utc::now()); + assert!(metadata.get_modified_at() <= &Utc::now()); + } + + #[test] + fn test_sync_type_default() { + assert_eq!(SyncType::default(), SyncType::RAFT); + } + + #[test] + fn test_metadata_serialization() { + let metadata = Metadata::new(EntityType::File); + let serialized = serde_json::to_string(&metadata).unwrap(); + let deserialized: Metadata = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(metadata, deserialized); + } +} diff --git a/monofs/lib/filesystem/mod.rs b/monofs/lib/filesystem/mod.rs new file mode 100644 index 0000000..4c11871 --- /dev/null +++ b/monofs/lib/filesystem/mod.rs @@ -0,0 +1,29 @@ +//! Filesystem implementation. + +mod dir; +mod entity; +mod eq; +mod error; +mod file; +mod kind; +mod link; +mod metadata; +mod resolvable; +mod softlink; +mod storeswitch; + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +pub use dir::*; +pub use entity::*; +pub use eq::*; +pub use error::*; +pub use file::*; +pub use kind::*; +pub use link::*; +pub use metadata::*; +pub use resolvable::*; +pub use softlink::*; +pub use storeswitch::*; diff --git a/monofs/lib/filesystem/resolvable.rs b/monofs/lib/filesystem/resolvable.rs new file mode 100644 index 0000000..35860c9 --- /dev/null +++ b/monofs/lib/filesystem/resolvable.rs @@ -0,0 +1,23 @@ +use futures::Future; +use monoutils_store::IpldStore; + +use crate::filesystem::FsResult; + +//-------------------------------------------------------------------------------------------------- +// Traits +//-------------------------------------------------------------------------------------------------- + +/// A trait for types that can be resolved to a target. +pub trait Resolvable<'a, S> +where + S: IpldStore, +{ + /// The target type that the resolvable type can be resolved to. + type Target: 'a; + + /// Resolves to a target type + fn resolve(&'a self, store: S) -> impl Future>; + + /// Resolves to a mutable target type + fn resolve_mut(&'a mut self, store: S) -> impl Future>; +} diff --git a/monofs/lib/filesystem/softlink.rs b/monofs/lib/filesystem/softlink.rs new file mode 100644 index 0000000..ee0979d --- /dev/null +++ b/monofs/lib/filesystem/softlink.rs @@ -0,0 +1,563 @@ +//! Symbolic link implementation. + +use std::{ + fmt::{self, Debug}, + sync::Arc, +}; + +use async_recursion::async_recursion; +use monoutils_store::{ + ipld::cid::Cid, IpldReferences, IpldStore, Storable, StoreError, StoreResult, +}; +use serde::{ + de::{self, DeserializeSeed}, + Deserialize, Deserializer, Serialize, +}; + +use crate::filesystem::{CidLink, Dir, EntityCidLink, File, FsError, FsResult, Metadata}; + +use super::{entity::Entity, kind::EntityType}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Represents a [`symbolic link`][softlink] to a file or directory in the `monofs` _immutable_ file system. +/// +/// ## Important +/// +/// Entities in `monofs` are designed to be immutable and clone-on-write meaning writes create +/// forks of the entity. +/// +/// [softlink]: https://en.wikipedia.org/wiki/Symbolic_link +#[derive(Clone)] +pub struct SoftLink +where + S: IpldStore, +{ + inner: Arc>, +} + +#[derive(Clone)] +struct SoftLinkInner +where + S: IpldStore, +{ + /// The metadata of the softlink. + metadata: Metadata, + + /// The store of the softlink. + store: S, + + /// The (weak) link to some target [`Entity`]. + // TODO: Because `SoftLink` refers to an entity by its Cid, it's behavior is a bit different from + // typical location-addressable file systems where softlinks break if the target entity is moved + // from its original location. `SoftLink` only breaks if the Cid to the target entity is deleted + // not the target entity itself. This is bad. + // + // In order to maintain compatibility with Unix-like systems, we may need to change this to an + // `EntityPathLink` in the future, where the path is relative to the location of the softlink. + link: EntityCidLink, +} + +/// Represents the result of following a softlink. +pub enum FollowResult<'a, S> +where + S: IpldStore, +{ + /// The softlink was successfully resolved to a non-softlink entity. + Resolved(&'a Entity), + + /// The maximum follow depth was reached without resolving to a non-softlink entity. + MaxDepthReached, + + /// A broken link was encountered during resolution. + BrokenLink(Cid), +} + +//-------------------------------------------------------------------------------------------------- +// Types: Serializable +//-------------------------------------------------------------------------------------------------- + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct SoftLinkSerializable { + metadata: Metadata, + target: Cid, +} + +pub(crate) struct SoftLinkDeserializeSeed { + pub(crate) store: S, +} + +//-------------------------------------------------------------------------------------------------- +// Methods: SoftLink +//-------------------------------------------------------------------------------------------------- + +impl SoftLink +where + S: IpldStore, +{ + /// Creates a new softlink. + pub fn with_cid(store: S, target: Cid) -> Self { + Self { + inner: Arc::new(SoftLinkInner { + metadata: Metadata::new(EntityType::SoftLink), + store, + link: CidLink::from(target), + }), + } + } + + /// Returns the metadata for the directory. + pub fn get_metadata(&self) -> &Metadata { + &self.inner.metadata + } + + /// Returns the store used to persist the softlink. + pub fn get_store(&self) -> &S { + &self.inner.store + } + + /// Gets the [`EntityCidLink`] of the target of the softlink. + pub fn get_link(&self) -> &EntityCidLink { + &self.inner.link + } + + /// Gets the [`Cid`] of the target of the softlink. + pub async fn get_cid(&self) -> FsResult + where + S: Send + Sync, + { + self.inner.link.resolve_cid().await + } + + /// Gets the [`Entity`] that the softlink points to. + pub async fn get_entity(&self) -> FsResult<&Entity> + where + S: Send + Sync, + { + self.inner + .link + .resolve_entity(self.inner.store.clone()) + .await + } + + /// Gets the [`Dir`] that the softlink points to. + pub async fn get_dir(&self) -> FsResult>> + where + S: Send + Sync, + { + match self.get_entity().await? { + Entity::Dir(dir) => Ok(Some(dir)), + _ => Ok(None), + } + } + + /// Gets the [`File`] that the softlink points to. + pub async fn get_file(&self) -> FsResult>> + where + S: Send + Sync, + { + match self.get_entity().await? { + Entity::File(file) => Ok(Some(file)), + _ => Ok(None), + } + } + + /// Gets the [`SoftLink`] that the softlink points to. + pub async fn get_softlink(&self) -> FsResult>> + where + S: Send + Sync, + { + match self.get_entity().await? { + Entity::SoftLink(softlink) => Ok(Some(softlink)), + _ => Ok(None), + } + } + + /// Deserializes to a `Dir` using an arbitrary deserializer and store. + pub fn deserialize_with<'de>( + deserializer: impl Deserializer<'de, Error: Into>, + store: S, + ) -> FsResult { + SoftLinkDeserializeSeed::new(store) + .deserialize(deserializer) + .map_err(Into::into) + } + + /// Tries to create a new `Dir` from a serializable representation. + pub(crate) fn try_from_serializable( + serializable: SoftLinkSerializable, + store: S, + ) -> FsResult { + Ok(SoftLink { + inner: Arc::new(SoftLinkInner { + metadata: serializable.metadata, + link: CidLink::from(serializable.target), + store, + }), + }) + } + + /// Follows the softlink to resolve the target entity. + /// + /// This method will follow the chain of softlinks up to the maximum depth specified in the metadata. + /// If the maximum depth is reached without resolving to a non-softlink entity, it returns `MaxDepthReached`. + /// If a broken link is encountered, it returns `BrokenLink`. + /// + /// ## Returns + /// + /// - `Ok(FollowResult::Resolved(entity))` if the softlink resolves to a non-softlink entity. + /// - `Ok(FollowResult::MaxDepthReached)` if the maximum follow depth is reached. + /// - `Ok(FollowResult::BrokenLink)` if a broken link is encountered. + /// - `Err(FsError)` if there's an error during the resolution process. + pub async fn follow(&self) -> FsResult> + where + S: Send + Sync, + { + let max_depth = *self.inner.metadata.get_softlink_depth(); + self.follow_recursive(max_depth).await + } + + #[async_recursion] + async fn follow_recursive(&self, remaining_depth: u32) -> FsResult> + where + S: Send + Sync, + { + if remaining_depth == 0 { + return Ok(FollowResult::MaxDepthReached); + } + + match self.get_entity().await { + Ok(entity) => match entity { + Entity::SoftLink(next_softlink) => { + next_softlink.follow_recursive(remaining_depth - 1).await + } + _ => Ok(FollowResult::Resolved(entity)), + }, + // We find the error `get_entity` returns that deals with not being able to load an entity + // from the store and return a `FollowResult::BrokenLink`. + Err(FsError::IpldStore(StoreError::Custom(any_err))) => { + if let Some(FsError::UnableToLoadEntity(cid)) = any_err.downcast::() { + return Ok(FollowResult::BrokenLink(*cid)); + } + + return Err(StoreError::custom(any_err).into()); + } + Err(e) => Err(e), + } + } + + /// Resolves the softlink to its target entity. + /// + /// This method will follow the chain of softlinks up to the maximum depth specified in the metadata. + /// It will return an error if the maximum depth is reached or if a broken link is encountered. + pub async fn resolve(&self) -> FsResult<&Entity> + where + S: Send + Sync, + { + match self.follow().await? { + FollowResult::Resolved(entity) => Ok(entity), + FollowResult::MaxDepthReached => Err(FsError::MaxFollowDepthReached), + FollowResult::BrokenLink(cid) => Err(FsError::BrokenSoftLink(cid)), + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Methods: FileDeserializeSeed +//-------------------------------------------------------------------------------------------------- + +impl SoftLinkDeserializeSeed { + fn new(store: S) -> Self { + Self { store } + } +} +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl From> for SoftLink +where + S: IpldStore + Clone, +{ + fn from(entity: Entity) -> Self { + Self { + inner: Arc::new(SoftLinkInner { + metadata: Metadata::new(EntityType::SoftLink), + store: entity.get_store().clone(), + link: EntityCidLink::from(entity), + }), + } + } +} + +impl From> for Entity +where + S: IpldStore + Clone, +{ + fn from(dir: Dir) -> Self { + Entity::Dir(dir) + } +} + +impl From> for Entity +where + S: IpldStore + Clone, +{ + fn from(file: File) -> Self { + Entity::File(file) + } +} + +impl From> for Entity +where + S: IpldStore + Clone, +{ + fn from(softlink: SoftLink) -> Self { + Entity::SoftLink(softlink) + } +} + +impl<'de, S> DeserializeSeed<'de> for SoftLinkDeserializeSeed +where + S: IpldStore, +{ + type Value = SoftLink; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let serializable = SoftLinkSerializable::deserialize(deserializer)?; + SoftLink::try_from_serializable(serializable, self.store).map_err(de::Error::custom) + } +} + +impl Storable for SoftLink +where + S: IpldStore + Send + Sync, +{ + async fn store(&self) -> StoreResult { + let serializable = SoftLinkSerializable { + metadata: self.inner.metadata.clone(), + target: self + .inner + .link + .resolve_cid() + .await + .map_err(StoreError::custom)?, + }; + + self.inner.store.put_node(&serializable).await + } + + async fn load(cid: &Cid, store: S) -> StoreResult { + let serializable = store.get_node(cid).await?; + SoftLink::try_from_serializable(serializable, store).map_err(StoreError::custom) + } +} + +impl Debug for SoftLink +where + S: IpldStore, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SoftLink") + .field("metadata", &self.inner.metadata) + .finish() + } +} + +impl IpldReferences for SoftLinkSerializable { + fn get_references<'a>(&'a self) -> Box + Send + 'a> { + // This empty because `SoftLink`s cannot have strong references to other entities. + Box::new(std::iter::empty()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + config::DEFAULT_SOFTLINK_DEPTH, + filesystem::{Dir, Entity, File}, + }; + use monoutils_store::MemoryStore; + + mod fixtures { + use super::*; + + pub async fn setup_test_env() -> (MemoryStore, Dir, File) { + let store = MemoryStore::default(); + let mut root_dir = Dir::new(store.clone()); + + let file_content = b"Hello, World!".to_vec(); + let file = File::with_content(store.clone(), file_content).await; + + root_dir.put_file("test_file.txt", file.clone()).unwrap(); + + (store, root_dir, file) + } + } + + #[tokio::test] + async fn test_softlink_creation() -> FsResult<()> { + let (store, root_dir, _) = fixtures::setup_test_env().await; + + let file_cid = root_dir + .get_entry("test_file.txt")? + .unwrap() + .resolve_cid() + .await?; + let softlink = SoftLink::with_cid(store, file_cid); + + assert_eq!( + softlink.get_metadata().get_entity_type(), + &EntityType::SoftLink + ); + assert_eq!(softlink.get_cid().await?, file_cid); + + Ok(()) + } + + #[tokio::test] + async fn test_softlink_from_entity() -> FsResult<()> { + let (_, _, file) = fixtures::setup_test_env().await; + + let file_entity = Entity::File(file); + let softlink = SoftLink::from(file_entity); + + assert_eq!( + softlink.get_metadata().get_entity_type(), + &EntityType::SoftLink + ); + assert!(matches!(softlink.get_entity().await?, Entity::File(_))); + + Ok(()) + } + + #[tokio::test] + async fn test_softlink_follow() -> FsResult<()> { + let (store, root_dir, _) = fixtures::setup_test_env().await; + + let file_cid = root_dir + .get_entry("test_file.txt")? + .unwrap() + .resolve_cid() + .await?; + let softlink = SoftLink::with_cid(store, file_cid); + + match softlink.follow().await? { + FollowResult::Resolved(entity) => { + assert!(matches!(entity, Entity::File(_))); + } + _ => panic!("Expected Resolved, got something else"), + } + + Ok(()) + } + + #[tokio::test] + async fn test_softlink_to_softlink() -> FsResult<()> { + let (store, root_dir, _) = fixtures::setup_test_env().await; + + let file_cid = root_dir + .get_entry("test_file.txt")? + .unwrap() + .resolve_cid() + .await?; + let softlink1 = SoftLink::with_cid(store.clone(), file_cid); + + let softlink1_cid = softlink1.store().await?; + let softlink2 = SoftLink::with_cid(store, softlink1_cid); + + match softlink2.follow().await? { + FollowResult::Resolved(entity) => { + assert!(matches!(entity, Entity::File(_))); + } + _ => panic!("Expected Resolved, got something else"), + } + + Ok(()) + } + + #[tokio::test] + async fn test_softlink_max_depth() -> FsResult<()> { + let (store, root_dir, _) = fixtures::setup_test_env().await; + + let file_cid = root_dir + .get_entry("test_file.txt")? + .unwrap() + .resolve_cid() + .await?; + + // Link depth 1 to file. + let mut softlink = SoftLink::with_cid(store.clone(), file_cid); + + // Link depth 9 to file. + for _ in 0..DEFAULT_SOFTLINK_DEPTH - 1 { + let cid = softlink.store().await?; + softlink = SoftLink::with_cid(store.clone(), cid); + } + + match softlink.follow().await? { + FollowResult::Resolved(entity) => { + assert!(matches!(entity, Entity::File(_))); + } + _ => panic!("Expected Resolved, got something else"), + } + + // Link depth 10 to file. + let cid = softlink.store().await?; + softlink = SoftLink::with_cid(store.clone(), cid); + + match softlink.follow().await? { + FollowResult::MaxDepthReached => {} + _ => panic!("Expected MaxDepthReached, got something else"), + } + + Ok(()) + } + + #[tokio::test] + async fn test_broken_softlink() -> FsResult<()> { + let store = MemoryStore::default(); + let non_existent_cid = Cid::default(); // This CID doesn't exist in the store + + let softlink = SoftLink::with_cid(store, non_existent_cid); + + match softlink.follow().await? { + FollowResult::BrokenLink(_) => {} + _ => panic!("Expected BrokenLink, got something else"), + } + + assert!(matches!( + softlink.resolve().await, + Err(FsError::BrokenSoftLink(_)) + )); + + Ok(()) + } + + #[tokio::test] + async fn test_softlink_resolve() -> FsResult<()> { + let (store, root_dir, _) = fixtures::setup_test_env().await; + + let file_cid = root_dir + .get_entry("test_file.txt")? + .unwrap() + .resolve_cid() + .await?; + let softlink = SoftLink::with_cid(store, file_cid); + + let resolved_entity = softlink.resolve().await?; + assert!(matches!(resolved_entity, Entity::File(_))); + + Ok(()) + } +} diff --git a/monofs/lib/filesystem/storeswitch.rs b/monofs/lib/filesystem/storeswitch.rs new file mode 100644 index 0000000..44cd05a --- /dev/null +++ b/monofs/lib/filesystem/storeswitch.rs @@ -0,0 +1,14 @@ +use monoutils_store::IpldStore; + +//-------------------------------------------------------------------------------------------------- +// Traits +//-------------------------------------------------------------------------------------------------- + +/// A trait for types that can be changed to a different store. +pub trait StoreSwitchable { + /// The type of the entity. + type WithStore; + + /// Change the store used to persist the entity. + fn change_store(self, new_store: U) -> Self::WithStore; +} diff --git a/monofs/lib/lib.rs b/monofs/lib/lib.rs index 40b7e7c..5410255 100644 --- a/monofs/lib/lib.rs +++ b/monofs/lib/lib.rs @@ -1,4 +1,4 @@ -//! `monofs` is a distributed, decentralized, secure file system. +//! `monofs` is an immutable distributed file system. #![warn(missing_docs)] #![allow(clippy::module_inception)] @@ -6,3 +6,8 @@ //-------------------------------------------------------------------------------------------------- // Exports //-------------------------------------------------------------------------------------------------- + +pub mod config; +pub mod filesystem; +pub mod store; +pub mod utils; diff --git a/monofs/lib/store/membufferstore.rs b/monofs/lib/store/membufferstore.rs new file mode 100644 index 0000000..2aae956 --- /dev/null +++ b/monofs/lib/store/membufferstore.rs @@ -0,0 +1,117 @@ +use std::{collections::HashSet, pin::Pin}; + +use bytes::Bytes; +use monoutils_store::{ + ipld::cid::Cid, Codec, DualStore, DualStoreConfig, IpldReferences, IpldStore, MemoryStore, + StoreResult, +}; +use serde::{de::DeserializeOwned, Serialize}; +use tokio::io::AsyncRead; + +//-------------------------------------------------------------------------------------------------- +// Types: MemoryBufferStore +//-------------------------------------------------------------------------------------------------- + +/// An [`IpldStore`][zeroutils_store::IpldStore] with two underlying stores: an ephemeral in-memory +/// store for writes and a user-provided store for back-up reads. +/// +/// This store is useful for creating a temporary buffer for writes +#[derive(Debug, Clone)] +pub struct MemoryBufferStore +where + S: IpldStore, +{ + inner: DualStore, +} + +//-------------------------------------------------------------------------------------------------- +// Methods: MemoryBufferStore +//-------------------------------------------------------------------------------------------------- + +impl MemoryBufferStore +where + S: IpldStore, +{ + /// Creates a new `MemoryBufferStore` with the given backup store. + pub fn new(backup_store: S) -> Self { + Self { + inner: DualStore::new( + MemoryStore::default(), + backup_store, + DualStoreConfig::default(), + ), + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl IpldStore for MemoryBufferStore +where + S: IpldStore + Sync, +{ + async fn put_node(&self, data: &T) -> StoreResult + where + T: Serialize + IpldReferences + Sync, + { + self.inner.put_node(data).await + } + + async fn put_bytes<'a>( + &'a self, + reader: impl AsyncRead + Send + Sync + 'a, + ) -> StoreResult { + self.inner.put_bytes(reader).await + } + + async fn put_raw_block(&self, bytes: impl Into + Send) -> StoreResult { + self.inner.put_raw_block(bytes).await + } + + async fn get_node(&self, cid: &Cid) -> StoreResult + where + T: DeserializeOwned + Send, + { + self.inner.get_node(cid).await + } + + async fn get_bytes<'a>( + &'a self, + cid: &'a Cid, + ) -> StoreResult>> { + self.inner.get_bytes(cid).await + } + + async fn get_raw_block(&self, cid: &Cid) -> StoreResult { + self.inner.get_raw_block(cid).await + } + + #[inline] + async fn has(&self, cid: &Cid) -> bool { + self.inner.has(cid).await + } + + fn get_supported_codecs(&self) -> HashSet { + self.inner.get_supported_codecs() + } + + #[inline] + fn get_node_block_max_size(&self) -> Option { + self.inner.get_node_block_max_size() + } + + #[inline] + fn get_raw_block_max_size(&self) -> Option { + self.inner.get_raw_block_max_size() + } + + async fn is_empty(&self) -> StoreResult { + self.inner.is_empty().await + } + + async fn get_size(&self) -> StoreResult { + self.inner.get_size().await + } +} diff --git a/monofs/lib/store/mod.rs b/monofs/lib/store/mod.rs new file mode 100644 index 0000000..5516ee9 --- /dev/null +++ b/monofs/lib/store/mod.rs @@ -0,0 +1,9 @@ +//! Stores for the filesystem. + +mod membufferstore; + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +pub use membufferstore::*; diff --git a/monofs/lib/utils/mod.rs b/monofs/lib/utils/mod.rs new file mode 100644 index 0000000..88e5778 --- /dev/null +++ b/monofs/lib/utils/mod.rs @@ -0,0 +1,7 @@ +//! Utility functions. + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +pub mod path; diff --git a/monofs/lib/utils/path.rs b/monofs/lib/utils/path.rs new file mode 100644 index 0000000..f682c14 --- /dev/null +++ b/monofs/lib/utils/path.rs @@ -0,0 +1,122 @@ +//! Path utilities. + +use typed_path::Utf8UnixPath; + +use crate::filesystem::{FsError, FsResult, Utf8UnixPathSegment}; + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Split the last component from a path. +pub fn split_last(path: &Utf8UnixPath) -> FsResult<(Option<&Utf8UnixPath>, Utf8UnixPathSegment)> { + // Root path are not allowed + if path.has_root() { + return Err(FsError::PathHasRoot(path.to_string())); + } + + // Empty paths are not allowed + if path.as_str().is_empty() { + return Err(FsError::PathIsEmpty); + } + + let filename = path + .file_name() + .ok_or_else(|| FsError::InvalidPathComponent(path.to_string()))? + .parse()?; + + let parent = path + .parent() + .and_then(|p| (!p.as_str().is_empty()).then_some(p)); + + Ok((parent, filename)) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use super::*; + use typed_path::Utf8UnixPathBuf; + + #[test] + fn test_split_last() -> FsResult<()> { + // Positive cases + assert_eq!( + split_last(&Utf8UnixPathBuf::from("foo/bar/baz"))?, + ( + Some(Utf8UnixPath::new("foo/bar")), + Utf8UnixPathSegment::from_str("baz")? + ) + ); + assert_eq!( + split_last(&Utf8UnixPathBuf::from("foo/bar"))?, + ( + Some(Utf8UnixPath::new("foo")), + Utf8UnixPathSegment::from_str("bar")? + ) + ); + assert_eq!( + split_last(&Utf8UnixPathBuf::from("baz"))?, + (None, Utf8UnixPathSegment::from_str("baz")?) + ); + + // Path with multiple components + assert_eq!( + split_last(&Utf8UnixPathBuf::from("foo/bar/baz"))?, + ( + Some(Utf8UnixPath::new("foo/bar")), + Utf8UnixPathSegment::from_str("baz")? + ) + ); + + // Path with spaces + assert_eq!( + split_last(&Utf8UnixPathBuf::from("path with/spaces in/file name"))?, + ( + Some(Utf8UnixPath::new("path with/spaces in")), + Utf8UnixPathSegment::from_str("file name")? + ) + ); + + // Unicode characters + assert_eq!( + split_last(&Utf8UnixPathBuf::from("路径/文件"))?, + ( + Some(Utf8UnixPath::new("路径")), + Utf8UnixPathSegment::from_str("文件")? + ) + ); + + // Path ending with slash + assert_eq!( + split_last(&Utf8UnixPathBuf::from("foo/bar/"))?, + ( + Some(Utf8UnixPath::new("foo")), + Utf8UnixPathSegment::from_str("bar")? + ) + ); + + // Negative cases + assert!(matches!( + split_last(&Utf8UnixPathBuf::from("")), + Err(FsError::PathIsEmpty) + )); + + assert!(matches!( + split_last(&Utf8UnixPathBuf::from("/")), + Err(FsError::PathHasRoot(_)) + )); + + assert!(matches!( + split_last(&Utf8UnixPathBuf::from("/foo")), + Err(FsError::PathHasRoot(_)) + )); + + Ok(()) + } +} diff --git a/monoutils-did/Cargo.toml b/monoutils-did/Cargo.toml new file mode 100644 index 0000000..98cf23c --- /dev/null +++ b/monoutils-did/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "monoutils-did" +version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "monoutils_did" +path = "lib/lib.rs" + +[dependencies] diff --git a/monoutils-did/lib/lib.rs b/monoutils-did/lib/lib.rs new file mode 100644 index 0000000..9d8b570 --- /dev/null +++ b/monoutils-did/lib/lib.rs @@ -0,0 +1,8 @@ +//! `monobase` is a distributed, decentralized, secure multi-model database. + +#![warn(missing_docs)] +#![allow(clippy::module_inception)] + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- diff --git a/monoutils-raft/Cargo.toml b/monoutils-raft/Cargo.toml new file mode 100644 index 0000000..4a9bfa2 --- /dev/null +++ b/monoutils-raft/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "monoutils-raft" +version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "monoutils_raft" +path = "lib/lib.rs" + +[dependencies] diff --git a/monoutils-raft/lib/lib.rs b/monoutils-raft/lib/lib.rs new file mode 100644 index 0000000..9d8b570 --- /dev/null +++ b/monoutils-raft/lib/lib.rs @@ -0,0 +1,8 @@ +//! `monobase` is a distributed, decentralized, secure multi-model database. + +#![warn(missing_docs)] +#![allow(clippy::module_inception)] + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- diff --git a/monoutils-store/Cargo.toml b/monoutils-store/Cargo.toml new file mode 100644 index 0000000..5bf898c --- /dev/null +++ b/monoutils-store/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "monoutils-store" +version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +edition.workspace = true +readme = "README.md" + +[lib] +name = "monoutils_store" +path = "lib/lib.rs" + +[dependencies] +aliasable = "0.1.3" +anyhow.workspace = true +async-stream.workspace = true +bytes.workspace = true +futures.workspace = true +hex = "0.4.3" +libipld = { workspace = true, features = ["serde-codec"] } +lru = "0.12.3" +serde = { workspace = true, features = ["derive"] } +serde_ipld_dagcbor = "0.6.1" +thiserror.workspace = true +tokio = { workspace = true, features = ["sync"] } +tokio-util = { workspace = true, features = ["io"] } diff --git a/monoutils-store/lib/chunker.rs b/monoutils-store/lib/chunker.rs new file mode 100644 index 0000000..c0b8a4e --- /dev/null +++ b/monoutils-store/lib/chunker.rs @@ -0,0 +1,23 @@ +use bytes::Bytes; +use futures::{stream::BoxStream, Future}; +use tokio::io::AsyncRead; + +use super::StoreResult; + +//-------------------------------------------------------------------------------------------------- +// Traits +//-------------------------------------------------------------------------------------------------- + +/// A chunker that splits incoming bytes into chunks and returns those chunks as a stream. +/// +/// This can be used by stores chunkers. +pub trait Chunker { + /// Chunks the given reader and returns a stream of bytes. + fn chunk<'a>( + &self, + reader: impl AsyncRead + Send + 'a, + ) -> impl Future>>> + Send; + + /// Returns the allowed maximum chunk size. If there is no limit, `None` is returned. + fn chunk_max_size(&self) -> Option; +} diff --git a/monoutils-store/lib/error.rs b/monoutils-store/lib/error.rs new file mode 100644 index 0000000..f79b98e --- /dev/null +++ b/monoutils-store/lib/error.rs @@ -0,0 +1,113 @@ +use std::{ + error::Error, + fmt::{self, Display}, +}; + +use libipld::Cid; +use thiserror::Error; + +use super::Codec; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// The result of a file system operation. +pub type StoreResult = Result; + +/// An error that occurred during a file system operation. +#[derive(Debug, Error, PartialEq)] +pub enum StoreError { + /// The block was not found. + #[error("Block not found: {0}")] + BlockNotFound(Cid), + + /// The node block is too large. + #[error("Node block too large: {0} > {1}")] + NodeBlockTooLarge(u64, u64), + + /// The raw block is too large. + #[error("Raw block too large: {0} > {1}")] + RawBlockTooLarge(u64, u64), + + /// Codec not supported. + #[error("Unsupported Codec: {0}")] + UnsupportedCodec(u64), + + /// Expected block codec does not match the actual codec. + #[error("Unexpected block codec: expected: {0:?} got: {1:?}")] + UnexpectedBlockCodec(Codec, Codec), + + /// Custom error. + #[error("Custom error: {0}")] + Custom(#[from] AnyError), + + /// Layout error. + #[error("Layout error: {0}")] + LayoutError(#[from] LayoutError), +} + +/// An error that occurred during a layout operation. +#[derive(Debug, Error, PartialEq)] +pub enum LayoutError { + /// No leaf block found. + #[error("No leaf block found")] + NoLeafBlock, +} + +/// An error that can represent any error. +#[derive(Debug)] +pub struct AnyError { + error: anyhow::Error, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl StoreError { + /// Creates a new `Err` result. + pub fn custom(error: impl Into) -> StoreError { + StoreError::Custom(AnyError { + error: error.into(), + }) + } +} + +impl AnyError { + /// Downcasts the error to a `T`. + pub fn downcast(&self) -> Option<&T> + where + T: Display + fmt::Debug + Send + Sync + 'static, + { + self.error.downcast_ref::() + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Creates an `Ok` `FsResult` d. +#[allow(non_snake_case)] +pub fn Ok(value: T) -> StoreResult { + Result::Ok(value) +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl PartialEq for AnyError { + fn eq(&self, other: &Self) -> bool { + self.error.to_string() == other.error.to_string() + } +} + +impl Display for AnyError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.error) + } +} + +impl Error for AnyError {} diff --git a/monoutils-store/lib/implementations/chunkers/constants.rs b/monoutils-store/lib/implementations/chunkers/constants.rs new file mode 100644 index 0000000..3094db2 --- /dev/null +++ b/monoutils-store/lib/implementations/chunkers/constants.rs @@ -0,0 +1,6 @@ +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +/// The default chunk size is 512 KiB. +pub const DEFAULT_CHUNK_MAX_SIZE: u64 = 512 * 1024; diff --git a/monoutils-store/lib/implementations/chunkers/fixed.rs b/monoutils-store/lib/implementations/chunkers/fixed.rs new file mode 100644 index 0000000..3a67b98 --- /dev/null +++ b/monoutils-store/lib/implementations/chunkers/fixed.rs @@ -0,0 +1,110 @@ +use std::pin::pin; + +use async_stream::try_stream; +use bytes::Bytes; +use futures::stream::BoxStream; +use tokio::io::{AsyncRead, AsyncReadExt}; + +use crate::{Chunker, StoreError, StoreResult}; + +use super::DEFAULT_CHUNK_MAX_SIZE; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// `FixedSizeChunker` splits data into fixed-size chunks, regardless of the content, in a simple +/// and deterministic. +#[derive(Clone, Debug)] +pub struct FixedSizeChunker { + /// The size of each chunk. + chunk_size: u64, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl FixedSizeChunker { + /// Creates a new `FixedSizeChunker` with the given `chunk_size`. + pub fn new(chunk_size: u64) -> Self { + Self { chunk_size } + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Chunker for FixedSizeChunker { + async fn chunk<'a>( + &self, + reader: impl AsyncRead + Send + 'a, + ) -> StoreResult>> { + let chunk_size = self.chunk_size; + + let s = try_stream! { + let reader = pin!(reader); + let mut chunk_reader = reader.take(chunk_size); // Derives a reader for reading the first chunk. + + loop { + let mut chunk = vec![]; + let n = chunk_reader.read_to_end(&mut chunk).await.map_err(StoreError::custom)?; + + if n == 0 { + break; + } + + yield Bytes::from(chunk); + + chunk_reader = chunk_reader.into_inner().take(chunk_size); // Derives a reader for reading the next chunk. + } + }; + + Ok(Box::pin(s)) + } + + fn chunk_max_size(&self) -> Option { + Some(self.chunk_size) + } +} + +impl Default for FixedSizeChunker { + fn default() -> Self { + Self::new(DEFAULT_CHUNK_MAX_SIZE) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use futures::StreamExt; + + use super::*; + + #[tokio::test] + async fn test_fixed_size_chunker() -> anyhow::Result<()> { + let data = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit."; + let chunker = FixedSizeChunker::new(10); + + let mut chunk_stream = chunker.chunk(&data[..]).await?; + let mut chunks = vec![]; + + while let Some(chunk) = chunk_stream.next().await { + chunks.push(chunk?); + } + + assert_eq!(chunks.len(), 6); + assert_eq!(chunks[0].to_vec(), b"Lorem ipsu"); + assert_eq!(chunks[1].to_vec(), b"m dolor si"); + assert_eq!(chunks[2].to_vec(), b"t amet, co"); + assert_eq!(chunks[3].to_vec(), b"nsectetur "); + assert_eq!(chunks[4].to_vec(), b"adipiscing"); + assert_eq!(chunks[5].to_vec(), b" elit."); + + Ok(()) + } +} diff --git a/monoutils-store/lib/implementations/chunkers/mod.rs b/monoutils-store/lib/implementations/chunkers/mod.rs new file mode 100644 index 0000000..2d33fba --- /dev/null +++ b/monoutils-store/lib/implementations/chunkers/mod.rs @@ -0,0 +1,11 @@ +mod constants; +mod fixed; +mod rabin; + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +pub use constants::*; +pub use fixed::*; +pub use rabin::*; diff --git a/monoutils-store/lib/implementations/chunkers/rabin.rs b/monoutils-store/lib/implementations/chunkers/rabin.rs new file mode 100644 index 0000000..44c70de --- /dev/null +++ b/monoutils-store/lib/implementations/chunkers/rabin.rs @@ -0,0 +1,57 @@ +use bytes::Bytes; +use futures::stream::BoxStream; +use tokio::io::AsyncRead; + +use crate::{Chunker, StoreResult}; + +use super::DEFAULT_CHUNK_MAX_SIZE; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A chunker that splits data into variable-size chunks using the Rabin fingerprinting algorithm. +/// +/// The `RabinChunker` leverages the Rabin fingerprinting technique to produce chunks of data with +/// variable sizes. This algorithm is particularly effective for identifying duplicate content within +/// files, as well as across different files, by creating consistent chunk boundaries. The resulting +/// chunks are then processed and stored in an IPLD form. +pub struct RabinChunker { + /// The size of each chunk. + chunk_size: u64, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl RabinChunker { + /// Creates a new `RabinChunker` with the given `chunk_size`. + pub fn new(chunk_size: u64) -> Self { + Self { chunk_size } + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Chunker for RabinChunker { + async fn chunk<'a>( + &self, + _reader: impl AsyncRead + Send + 'a, + ) -> StoreResult>> { + let _ = _reader; + todo!() // TODO: To be implemented + } + + fn chunk_max_size(&self) -> Option { + Some(self.chunk_size) + } +} + +impl Default for RabinChunker { + fn default() -> Self { + Self::new(DEFAULT_CHUNK_MAX_SIZE) + } +} diff --git a/monoutils-store/lib/implementations/layouts/balanced.rs b/monoutils-store/lib/implementations/layouts/balanced.rs new file mode 100644 index 0000000..5d54c27 --- /dev/null +++ b/monoutils-store/lib/implementations/layouts/balanced.rs @@ -0,0 +1,41 @@ +use std::pin::Pin; + +use bytes::Bytes; +use futures::stream::BoxStream; +use libipld::Cid; +use tokio::io::AsyncRead; + +use crate::{IpldStore, Layout, StoreResult}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A layout that organizes data into a balanced DAG. +#[derive(Clone, Debug, PartialEq)] +pub struct BalancedDagLayout { + /// The maximum number of children each node can have. + degree: usize, +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Layout for BalancedDagLayout { + async fn organize<'a>( + &self, + _stream: BoxStream<'a, StoreResult>, + _store: impl IpldStore + Send + 'a, + ) -> StoreResult>> { + todo!() // TODO: To be implemented + } + + async fn retrieve<'a>( + &self, + _cid: &Cid, + _store: impl IpldStore + Send + 'a, + ) -> StoreResult>> { + todo!() // TODO: To be implemented + } +} diff --git a/monoutils-store/lib/implementations/layouts/flat.rs b/monoutils-store/lib/implementations/layouts/flat.rs new file mode 100644 index 0000000..60ec347 --- /dev/null +++ b/monoutils-store/lib/implementations/layouts/flat.rs @@ -0,0 +1,547 @@ +use std::{ + cmp::Ordering, + io::{Error, ErrorKind, SeekFrom}, + pin::Pin, + task::{Context, Poll}, +}; + +use aliasable::boxed::AliasableBox; +use async_stream::try_stream; +use bytes::Bytes; +use futures::{ready, stream::BoxStream, Future, StreamExt}; +use libipld::Cid; +use tokio::io::{AsyncRead, AsyncSeek, ReadBuf}; + +use crate::{ + IpldStore, Layout, LayoutError, LayoutSeekable, MerkleNode, SeekableReader, StoreError, + StoreResult, +}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A layout that organizes data into a flat array of chunks with a single merkle node parent. +/// +/// ```txt +/// ┌─────────────┐ +/// │ Merkle Node │ +/// └──────┬──────┘ +/// │ +/// ┌───────────────┬──────┴────────┬─────────────────┐ +/// │ │ │ │ +/// 0 ▼ 1 ▼ 2 ▼ 3 ▼ +/// ┌──┬──┬──┐ ┌──┬──┬──┬──┬──┐ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┬──┐ +/// │0 │1 │2 │ │3 │4 │5 │6 │7 │ │8 │9 │10│11│ │12│13│14│15│16│17│ +/// └──┴──┴──┘ └──┴──┴──┴──┴──┘ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┴──┘ +/// ``` +#[derive(Clone, Debug, PartialEq, Default)] +pub struct FlatLayout {} + +/// A reader for the flat DAG layout. +/// +/// The reader maintains three state variables: +/// +/// - The current byte position, `byte_cursor`. +/// - The index of the current chunk within the node's children array, `chunk_index`. +/// - The distance (in bytes) of the current chunk index from the start, `chunk_distance`. +/// +/// These state variables are used to determine the current chunk to read from and the byte position +/// within the chunk to read from. It basically enables seeking to any byte position within the +/// chunk array. +/// +/// ```txt +/// Chunk Index = 1 +/// Chunk Distance = 3 +/// │ +/// │ +/// 0 ▼ 1 2 +/// ┌──┬──┬──┐ ┌──┬──┬──┬──┬──┐ ┌──┬──┬──┬──┐ +/// │0 │1 │2 │ │3 │4 │5 │6 │7 │ │8 │9 │10│11│ +/// └──┴──┴──┘ └──┴──┴──┴──┴──┘ └──┴──┴──┴──┘ +/// ▲ +/// │ +/// │ +/// Byte Cursor = 5 +/// ``` +pub struct FlatLayoutReader +where + S: IpldStore, +{ + /// The current byte position. + byte_cursor: u64, + + /// The index of the current chunk within the node's children array. + chunk_index: u64, + + /// The distance (in bytes) of the current chunk index from the start. + chunk_distance: u64, + + /// A function to get a raw block. + /// + /// ## Important + /// + /// Holds a reference to other fields in this struct. Declared first to ensure it is dropped + /// before the other fields. + get_raw_block_fn: Pin> + Send + Sync + 'static>>, + + /// The store associated with the reader. + /// + /// ## Warning + /// + /// Field must not be moved as it is referenced by `get_raw_block_fn`. + store: AliasableBox, + + /// The node that the reader is reading from. + /// + /// ## Warning + /// + /// Field must not be moved as it is referenced by `get_raw_block_fn`. + node: AliasableBox, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl FlatLayout { + /// Create a new flat DAG layout. + pub fn new() -> Self { + FlatLayout {} + } +} + +impl FlatLayoutReader +where + S: IpldStore + Sync, +{ + /// Create a new flat DAG reader. + fn new(node: MerkleNode, store: S) -> StoreResult { + // Store node and store in the heap and make them aliasable. + let node = AliasableBox::from_unique(Box::new(node)); + let store = AliasableBox::from_unique(Box::new(store)); + + // Create future to get the first node child. + let get_raw_block_fn: Pin> + Send + Sync>> = + Box::pin( + store.get_raw_block( + node.children + .first() + .map(|(cid, _)| cid) + .ok_or(StoreError::from(LayoutError::NoLeafBlock))?, + ), + ); + + // Unsafe magic to escape Rust ownership grip. + let get_raw_block_fn: Pin< + Box> + Send + Sync + 'static>, + > = unsafe { std::mem::transmute(get_raw_block_fn) }; + + Ok(FlatLayoutReader { + byte_cursor: 0, + chunk_index: 0, + chunk_distance: 0, + get_raw_block_fn, + node, + store, + }) + } + + fn fix_future(&mut self) { + // Create future to get the next child. + let get_raw_block_fn: Pin> + Send + Sync>> = + Box::pin(async { + let bytes = self + .store + .get_raw_block( + self.node + .children + .get(self.chunk_index as usize) + .map(|(cid, _)| cid) + .ok_or(StoreError::from(LayoutError::NoLeafBlock))?, + ) + .await?; + + // We just need bytes starting from byte cursor. + let bytes = Bytes::copy_from_slice( + &bytes[(self.byte_cursor - self.chunk_distance) as usize..], + ); + + Ok(bytes) + }); + + // Unsafe magic to escape Rust ownership grip. + let get_raw_block_fn: Pin< + Box> + Send + Sync + 'static>, + > = unsafe { std::mem::transmute(get_raw_block_fn) }; + + // Update type's future. + self.get_raw_block_fn = get_raw_block_fn; + } + + fn read_update(&mut self, left_over: &[u8], consumed: u64) -> StoreResult<()> { + // Update the byte cursor. + self.byte_cursor += consumed; + + // If there's left over bytes, we create a future to return the left over bytes. + if !left_over.is_empty() { + let bytes = Bytes::copy_from_slice(left_over); + let get_raw_block_fn = Box::pin(async { Ok(bytes) }); + self.get_raw_block_fn = get_raw_block_fn; + return Ok(()); + } + + // If we've reached the end of the bytes, create a future that returns empty bytes. + if self.byte_cursor >= self.node.size as u64 { + let get_raw_block_fn = Box::pin(async { Ok(Bytes::new()) }); + self.get_raw_block_fn = get_raw_block_fn; + return Ok(()); + } + + // Update the chunk distance and chunk index. + self.chunk_distance += self.node.children[self.chunk_index as usize].1 as u64; + self.chunk_index += 1; + + // Update the future. + self.fix_future(); + + Ok(()) + } + + fn seek_update(&mut self, byte_cursor: u64) -> StoreResult<()> { + // Update the byte cursor. + self.byte_cursor = byte_cursor; + + // If we've reached the end of the bytes, create a future that returns empty bytes. + if self.byte_cursor >= self.node.size as u64 { + let get_raw_block_fn = Box::pin(async { Ok(Bytes::new()) }); + self.get_raw_block_fn = get_raw_block_fn; + return Ok(()); + } + + // We need to update the chunk index and distance essentially making sure that chunk index and distance + // are referring to the chunk that the byte cursor is pointing to. + loop { + match self.chunk_distance.cmp(&byte_cursor) { + Ordering::Less => { + if self.chunk_distance + self.node.children[self.chunk_index as usize].1 as u64 + > byte_cursor + { + break; + } + + self.chunk_distance += self.node.children[self.chunk_index as usize].1 as u64; + self.chunk_index += 1; + + continue; + } + Ordering::Greater => { + self.chunk_distance -= self.node.children[self.chunk_index as usize].1 as u64; + self.chunk_index -= 1; + + continue; + } + _ => break, + } + } + + // Update the future. + self.fix_future(); + + Ok(()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Layout for FlatLayout { + async fn organize<'a>( + &self, + mut stream: BoxStream<'a, StoreResult>, + store: impl IpldStore + Send + 'a, + ) -> StoreResult>> { + let s = try_stream! { + let mut children = Vec::new(); + while let Some(Ok(chunk)) = stream.next().await { + let len = chunk.len(); + let cid = store.put_raw_block(chunk).await?; + children.push((cid, len)); + yield cid; + } + + let node = MerkleNode::new(children); + let cid = store.put_node(&node).await?; + + yield cid; + }; + + Ok(Box::pin(s)) + } + + async fn retrieve<'a>( + &self, + cid: &Cid, + store: impl IpldStore + Send + Sync + 'a, + ) -> StoreResult>> { + let node = store.get_node(cid).await?; + let reader = FlatLayoutReader::new(node, store)?; + Ok(Box::pin(reader)) + } +} + +impl LayoutSeekable for FlatLayout { + async fn retrieve_seekable<'a>( + &self, + cid: &'a Cid, + store: impl IpldStore + Send + Sync + 'a, + ) -> StoreResult>> { + let node = store.get_node(cid).await?; + let reader = FlatLayoutReader::new(node, store)?; + Ok(Box::pin(reader)) + } +} + +impl AsyncRead for FlatLayoutReader +where + S: IpldStore + Sync, +{ + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + // Get the next chunk of bytes. + let bytes = ready!(self.get_raw_block_fn.as_mut().poll(cx)) + .map_err(|e| Error::new(ErrorKind::Other, e))?; + + // If the bytes is longer than the buffer, we only take the amount that fits. + let (taken, left_over) = if bytes.len() > buf.remaining() { + bytes.split_at(buf.remaining()) + } else { + (&bytes[..], &[][..]) + }; + + // Copy the slice to the buffer. + buf.put_slice(taken); + + // Update the reader's state. + self.read_update(left_over, taken.len() as u64) + .map_err(|e| Error::new(ErrorKind::Other, e))?; + + Poll::Ready(Ok(())) + } +} + +impl AsyncSeek for FlatLayoutReader +where + S: IpldStore + Sync, +{ + fn start_seek(mut self: Pin<&mut Self>, position: SeekFrom) -> std::io::Result<()> { + let byte_cursor = match position { + SeekFrom::Start(offset) => { + if offset >= self.node.size as u64 { + return Err(Error::new( + ErrorKind::InvalidInput, + "Seek from start position out of bounds", + )); + } + + offset + } + SeekFrom::Current(offset) => { + let new_cursor = self.byte_cursor as i64 + offset; + if new_cursor < 0 || new_cursor >= self.node.size as i64 { + return Err(Error::new( + ErrorKind::InvalidInput, + "Seek from current position out of bounds", + )); + } + + new_cursor as u64 + } + SeekFrom::End(offset) => { + let new_cursor = self.node.size as i64 + offset; + if new_cursor < 0 || new_cursor >= self.node.size as i64 { + return Err(Error::new( + ErrorKind::InvalidInput, + "Seek from end position out of bounds", + )); + } + + new_cursor as u64 + } + }; + + // Update the reader's state. + self.seek_update(byte_cursor) + .map_err(|e| Error::new(ErrorKind::Other, e))?; + + Ok(()) + } + + fn poll_complete(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(self.byte_cursor)) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use futures::TryStreamExt; + use tokio::io::{AsyncReadExt, AsyncSeekExt}; + + use crate::MemoryStore; + + use super::*; + + #[tokio::test] + async fn test_flat_dag_layout_organize_and_retrieve() -> anyhow::Result<()> { + let store = MemoryStore::default(); + let (data, _, chunk_stream) = fixtures::data_and_chunk_stream(); + + // Organize chunks into a DAG. + let layout = FlatLayout::default(); + let cid_stream = layout.organize(chunk_stream, store.clone()).await?; + + // Get the CID of the merkle node. + let cids = cid_stream.try_collect::>().await?; + let cid = cids.last().unwrap(); + + // Case: fill buffer automatically with `read_to_end` + let mut reader = layout.retrieve(cid, store.clone()).await?; + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).await?; + + assert_eq!(bytes, data); + + // Case: fill buffer manually with `read` + let mut reader = layout.retrieve(cid, store).await?; + let mut bytes: Vec = vec![]; + loop { + let mut buf = vec![0; 5]; + let filled = reader.read(&mut buf).await?; + if filled == 0 { + break; + } + + bytes.extend(&buf[..filled]); + } + + assert_eq!(bytes, data); + + Ok(()) + } + + #[tokio::test] + async fn test_flat_dag_layout_seek() -> anyhow::Result<()> { + let store = MemoryStore::default(); + let (_, chunks, chunk_stream) = fixtures::data_and_chunk_stream(); + + // Organize chunks into a DAG. + let layout = FlatLayout::default(); + let cid_stream = layout.organize(chunk_stream, store.clone()).await?; + + // Get the CID of the first chunk. + let cids = cid_stream.try_collect::>().await?; + let cid = cids.last().unwrap(); + + // Get seekable reader. + let mut reader = layout.retrieve_seekable(cid, store).await?; + + // Case: read the first chunk. + let mut buf = vec![0; 5]; + reader.read(&mut buf).await?; + + assert_eq!(buf, chunks[0]); + + // Case: skip a chunk by seeking from current and have cursor be at boundary of chunk. + let mut buf = vec![0; 5]; + reader.seek(SeekFrom::Current(5)).await?; + reader.read(&mut buf).await?; + + assert_eq!(buf, chunks[2]); + + // Case: seek to the next chunk from current and have cursor be in the middle of chunk. + let mut buf = vec![0; 3]; + reader.seek(SeekFrom::Current(3)).await?; + reader.read(&mut buf).await?; + + assert_eq!(buf, chunks[3][3..]); + + // Case: Seek to some chunk before end. + let mut buf = vec![0; 5]; + reader.seek(SeekFrom::End(-5)).await?; + reader.read(&mut buf).await?; + + assert_eq!(buf, chunks[9]); + + // Case: Seek to some chunk after start. + let mut buf = vec![0; 5]; + reader.seek(SeekFrom::Start(5)).await?; + reader.read(&mut buf).await?; + + assert_eq!(buf, chunks[1]); + + // Case: Fail: Seek beyond end. + let result = reader.seek(SeekFrom::End(5)).await; + assert!(result.is_err()); + + let result = reader.seek(SeekFrom::End(0)).await; + assert!(result.is_err()); + + let result = reader.seek(SeekFrom::Start(100)).await; + assert!(result.is_err()); + + let result = reader.seek(SeekFrom::Current(100)).await; + assert!(result.is_err()); + + // Case: Fail: Seek before start. + let result = reader.seek(SeekFrom::Current(-100)).await; + assert!(result.is_err()); + + Ok(()) + } +} + +#[cfg(test)] +mod fixtures { + use futures::{stream, Stream}; + + use super::*; + + pub(super) fn data_and_chunk_stream() -> ( + [u8; 56], + Vec, + Pin> + Send + 'static>>, + ) { + let data = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit.".to_owned(); + + let chunks = vec![ + Bytes::from("Lorem"), + Bytes::from(" ipsu"), + Bytes::from("m dol"), + Bytes::from("or sit"), + Bytes::from(" amet,"), + Bytes::from(" conse"), + Bytes::from("ctetur"), + Bytes::from(" adipi"), + Bytes::from("scing "), + Bytes::from("elit."), + ]; + + let chunks_result = chunks + .iter() + .cloned() + .map(|b| crate::Ok(b)) + .collect::>(); + + let chunk_stream = Box::pin(stream::iter(chunks_result)); + + (data, chunks, chunk_stream) + } +} diff --git a/monoutils-store/lib/implementations/layouts/mod.rs b/monoutils-store/lib/implementations/layouts/mod.rs new file mode 100644 index 0000000..e10c660 --- /dev/null +++ b/monoutils-store/lib/implementations/layouts/mod.rs @@ -0,0 +1,11 @@ +mod balanced; +mod flat; +mod trickle; + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +// pub use balanced::*; +pub use flat::*; +// pub use trickle::*; diff --git a/monoutils-store/lib/implementations/layouts/trickle.rs b/monoutils-store/lib/implementations/layouts/trickle.rs new file mode 100644 index 0000000..71a97cf --- /dev/null +++ b/monoutils-store/lib/implementations/layouts/trickle.rs @@ -0,0 +1,38 @@ +use std::pin::Pin; + +use bytes::Bytes; +use futures::stream::BoxStream; +use libipld::Cid; +use tokio::io::AsyncRead; + +use crate::{IpldStore, Layout, StoreResult}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A layout that organizes data into a trickle DAG. +#[derive(Clone, Debug, PartialEq)] +pub struct TrickleDagLayout {} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Layout for TrickleDagLayout { + async fn organize<'a>( + &self, + _stream: BoxStream<'a, StoreResult>, + _store: impl IpldStore + Send + Sync + 'a, + ) -> StoreResult>> { + todo!() // TODO: To be implemented + } + + async fn retrieve<'a>( + &self, + _cid: &Cid, + _store: impl IpldStore + Send + Sync + 'a, + ) -> StoreResult>> { + todo!() // TODO: To be implemented + } +} diff --git a/monoutils-store/lib/implementations/mod.rs b/monoutils-store/lib/implementations/mod.rs new file mode 100644 index 0000000..6ee24d9 --- /dev/null +++ b/monoutils-store/lib/implementations/mod.rs @@ -0,0 +1,11 @@ +mod chunkers; +mod layouts; +mod stores; + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +pub use chunkers::*; +pub use layouts::*; +pub use stores::*; diff --git a/monoutils-store/lib/implementations/stores/dualstore.rs b/monoutils-store/lib/implementations/stores/dualstore.rs new file mode 100644 index 0000000..f9a0bca --- /dev/null +++ b/monoutils-store/lib/implementations/stores/dualstore.rs @@ -0,0 +1,278 @@ +use std::{collections::HashSet, pin::Pin}; + +use bytes::Bytes; +use libipld::Cid; +use serde::{de::DeserializeOwned, Serialize}; +use tokio::io::AsyncRead; + +use crate::{Codec, IpldReferences, IpldStore, StoreError, StoreResult}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A dual store that stores blocks on two different stores. +#[derive(Debug, Clone)] +pub struct DualStore +where + A: IpldStore, + B: IpldStore, +{ + store_a: A, + store_b: B, + config: DualStoreConfig, +} + +/// Choices for selecting which store to use for a given operation. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Choice { + /// Use the first store. + A, + /// Use the second store. + B, +} + +/// Configuration for a dual store. +#[derive(Debug, Clone)] +pub struct DualStoreConfig { + /// The default store to use. + pub default: Choice, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl DualStore +where + A: IpldStore, + B: IpldStore, +{ + /// Creates a new dual store from two stores. + pub fn new(store_a: A, store_b: B, config: DualStoreConfig) -> Self { + Self { + store_a, + store_b, + config, + } + } + + /// Gets the type stored as an IPLD data from a chosen store by its `Cid`. + pub async fn get_node_from(&self, cid: &Cid, choice: Choice) -> StoreResult + where + D: DeserializeOwned + Send, + { + match choice { + Choice::A => self.store_a.get_node(cid).await, + Choice::B => self.store_b.get_node(cid).await, + } + } + + /// Gets the bytes stored in a chosen store as raw bytes by its `Cid`. + pub async fn get_bytes_from<'a>( + &'a self, + cid: &'a Cid, + choice: Choice, + ) -> StoreResult>> { + match choice { + Choice::A => self.store_a.get_bytes(cid).await, + Choice::B => self.store_b.get_bytes(cid).await, + } + } + + /// Gets raw bytes from a chosen store as a single block by its `Cid`. + pub async fn get_raw_block_from(&self, cid: &Cid, choice: Choice) -> StoreResult { + match choice { + Choice::A => self.store_a.get_raw_block(cid).await, + Choice::B => self.store_b.get_raw_block(cid).await, + } + } + + /// Saves a serializable type to a chosen store and returns the `Cid` to it. + pub async fn put_node_into(&self, data: &T, choice: Choice) -> StoreResult + where + T: Serialize + IpldReferences + Sync, + { + match choice { + Choice::A => self.store_a.put_node(data).await, + Choice::B => self.store_b.put_node(data).await, + } + } + + /// Saves raw bytes to a chosen store and returns the `Cid` to it. + pub async fn put_bytes_into( + &self, + bytes: impl AsyncRead + Send + Sync, + choice: Choice, + ) -> StoreResult { + match choice { + Choice::A => self.store_a.put_bytes(bytes).await, + Choice::B => self.store_b.put_bytes(bytes).await, + } + } + + /// Saves raw bytes as a single block to a chosen store and returns the `Cid` to it. + pub async fn put_raw_block_into( + &self, + bytes: impl Into + Send, + choice: Choice, + ) -> StoreResult { + match choice { + Choice::A => self.store_a.put_raw_block(bytes).await, + Choice::B => self.store_b.put_raw_block(bytes).await, + } + } + + /// Checks if a block exists in a chosen store by its `Cid`. + pub async fn has_from(&self, cid: &Cid, choice: Choice) -> bool { + match choice { + Choice::A => self.store_a.has(cid).await, + Choice::B => self.store_b.has(cid).await, + } + } +} + +impl Choice { + /// Returns the other choice. + pub fn other(&self) -> Self { + match self { + Choice::A => Choice::B, + Choice::B => Choice::A, + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl IpldStore for DualStore +where + A: IpldStore + Sync, + B: IpldStore + Sync, +{ + async fn put_node(&self, data: &T) -> StoreResult + where + T: Serialize + IpldReferences + Sync, + { + self.put_node_into(data, self.config.default).await + } + + async fn put_bytes<'a>(&'a self, bytes: impl AsyncRead + Send + Sync + 'a) -> StoreResult { + self.put_bytes_into(bytes, self.config.default).await + } + + async fn put_raw_block(&self, bytes: impl Into + Send) -> StoreResult { + self.put_raw_block_into(bytes, self.config.default).await + } + + async fn get_node(&self, cid: &Cid) -> StoreResult + where + D: DeserializeOwned + Send, + { + match self.get_node_from(cid, self.config.default).await { + Ok(data) => Ok(data), + Err(StoreError::BlockNotFound(_)) => { + let choice = self.config.default.other(); + self.get_node_from(cid, choice).await + } + Err(err) => Err(err), + } + } + + async fn get_bytes<'a>( + &'a self, + cid: &'a Cid, + ) -> StoreResult>> { + match self.get_bytes_from(cid, self.config.default).await { + Ok(bytes) => Ok(bytes), + Err(StoreError::BlockNotFound(_)) => { + let choice = self.config.default.other(); + self.get_bytes_from(cid, choice).await + } + Err(err) => Err(err), + } + } + + async fn get_raw_block(&self, cid: &Cid) -> StoreResult { + match self.get_raw_block_from(cid, self.config.default).await { + Ok(bytes) => Ok(bytes), + Err(StoreError::BlockNotFound(_)) => { + let choice = self.config.default.other(); + self.get_raw_block_from(cid, choice).await + } + Err(err) => Err(err), + } + } + + async fn has(&self, cid: &Cid) -> bool { + match self.has_from(cid, self.config.default).await { + true => true, + false => self.has_from(cid, self.config.default.other()).await, + } + } + + fn get_supported_codecs(&self) -> HashSet { + self.store_a + .get_supported_codecs() + .into_iter() + .chain(self.store_b.get_supported_codecs()) + .collect() + } + + fn get_node_block_max_size(&self) -> Option { + self.store_a + .get_node_block_max_size() + .max(self.store_b.get_node_block_max_size()) + } + + fn get_raw_block_max_size(&self) -> Option { + self.store_a + .get_raw_block_max_size() + .max(self.store_b.get_raw_block_max_size()) + } + + async fn is_empty(&self) -> StoreResult { + Ok(self.store_a.is_empty().await? && self.store_b.is_empty().await?) + } + + async fn get_size(&self) -> StoreResult { + Ok(self.store_a.get_size().await? + self.store_b.get_size().await?) + } +} + +impl Default for DualStoreConfig { + fn default() -> Self { + Self { default: Choice::A } + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use crate::MemoryStore; + + use super::*; + + #[tokio::test] + async fn test_dual_store_put_and_get() -> anyhow::Result<()> { + let store_a = MemoryStore::default(); + let store_b = MemoryStore::default(); + let dual_store = DualStore::new(store_a, store_b, Default::default()); + + let cid_0 = dual_store.put_node_into(&"hello", Choice::A).await?; + let cid_1 = dual_store.put_node_into(&250, Choice::B).await?; + let cid_2 = dual_store.put_node_into(&"world", Choice::A).await?; + let cid_3 = dual_store.put_node_into(&500, Choice::B).await?; + + assert_eq!(dual_store.get_node::(&cid_0).await?, "hello"); + assert_eq!(dual_store.get_node::(&cid_1).await?, 250); + assert_eq!(dual_store.get_node::(&cid_2).await?, "world"); + assert_eq!(dual_store.get_node::(&cid_3).await?, 500); + + Ok(()) + } +} diff --git a/monoutils-store/lib/implementations/stores/memstore.rs b/monoutils-store/lib/implementations/stores/memstore.rs new file mode 100644 index 0000000..ca98909 --- /dev/null +++ b/monoutils-store/lib/implementations/stores/memstore.rs @@ -0,0 +1,314 @@ +use std::{ + collections::{HashMap, HashSet}, + pin::Pin, + sync::Arc, +}; + +use bytes::Bytes; +use futures::StreamExt; +use libipld::Cid; +use serde::{de::DeserializeOwned, Serialize}; +use tokio::{io::AsyncRead, sync::RwLock}; + +use crate::{ + utils, Chunker, Codec, FixedSizeChunker, FlatLayout, IpldReferences, IpldStore, + IpldStoreSeekable, Layout, LayoutSeekable, SeekableReader, StoreError, StoreResult, +}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// An in-memory storage for IPLD node and raw blocks with reference counting. +/// +/// This store maintains a reference count for each stored block. Reference counting is used to +/// determine when a block can be safely removed from the store. +#[derive(Debug, Clone)] +// TODO: Use BalancedDagLayout as default +pub struct MemoryStore +where + C: Chunker, + L: Layout, +{ + /// Represents the blocks stored in the store. + /// + /// When data is added to the store, it may not necessarily fit into the acceptable block size + /// limit, so it is chunked into smaller blocks. + /// + /// The `usize` is used for counting the references to blocks within the store. + blocks: Arc>>, + + /// The chunking algorithm used to split data into chunks. + chunker: C, + + /// The layout strategy used to store chunked data. + layout: L, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl MemoryStore +where + C: Chunker, + L: Layout, +{ + /// Creates a new `MemoryStore` with the given `chunker` and `layout`. + pub fn new(chunker: C, layout: L) -> Self { + MemoryStore { + blocks: Arc::new(RwLock::new(HashMap::new())), + chunker, + layout, + } + } + + /// Prints all the blocks in the store. + pub fn debug(&self) + where + C: Clone + Send, + L: Clone + Send, + { + let store = self.clone(); + tokio::spawn(async move { + let blocks = store.blocks.read().await; + for (cid, (size, bytes)) in blocks.iter() { + println!("\ncid: {} ({:?})\nkey: {}", cid, size, hex::encode(bytes)); + } + }); + } + + /// Increments the reference count of the blocks with the given `Cid`s. + async fn inc_refs(&self, cids: impl Iterator) { + for cid in cids { + if let Some((size, _)) = self.blocks.write().await.get_mut(cid) { + *size += 1; + } + } + } + + /// Stores raw bytes in the store without any size checks. + async fn store_raw(&self, bytes: Bytes, codec: Codec) -> Cid { + let cid = utils::make_cid(codec, &bytes); + self.blocks.write().await.insert(cid, (1, bytes)); + cid + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl IpldStore for MemoryStore +where + C: Chunker + Clone + Send + Sync, + L: Layout + Clone + Send + Sync, +{ + async fn put_node(&self, data: &T) -> StoreResult + where + T: Serialize + IpldReferences + Sync, + { + // Serialize the data to bytes. + let bytes = Bytes::from(serde_ipld_dagcbor::to_vec(&data).map_err(StoreError::custom)?); + + // Check if the data exceeds the node maximum block size. + if let Some(max_size) = self.get_node_block_max_size() { + if bytes.len() as u64 > max_size { + return Err(StoreError::NodeBlockTooLarge(bytes.len() as u64, max_size)); + } + } + + // Increment the reference count of the block. + self.inc_refs(data.get_references()).await; + + Ok(self.store_raw(bytes, Codec::DagCbor).await) + } + + async fn put_bytes<'a>( + &'a self, + reader: impl AsyncRead + Send + Sync + 'a, + ) -> StoreResult { + let chunk_stream = self.chunker.chunk(reader).await?; + let mut cid_stream = self.layout.organize(chunk_stream, self.clone()).await?; + + // Take the last `Cid` from the stream. + let mut cid = cid_stream.next().await.unwrap()?; + while let Some(result) = cid_stream.next().await { + cid = result?; + } + + Ok(cid) + } + + async fn put_raw_block(&self, bytes: impl Into) -> StoreResult { + let bytes = bytes.into(); + if let Some(max_size) = self.get_raw_block_max_size() { + if bytes.len() as u64 > max_size { + return Err(StoreError::RawBlockTooLarge(bytes.len() as u64, max_size)); + } + } + + Ok(self.store_raw(bytes, Codec::Raw).await) + } + + async fn get_node(&self, cid: &Cid) -> StoreResult + where + T: DeserializeOwned, + { + let blocks = self.blocks.read().await; + match blocks.get(cid) { + Some((_, bytes)) => match cid.codec().try_into()? { + Codec::DagCbor => { + let data = + serde_ipld_dagcbor::from_slice::(bytes).map_err(StoreError::custom)?; + Ok(data) + } + codec => Err(StoreError::UnexpectedBlockCodec(Codec::DagCbor, codec)), + }, + None => Err(StoreError::BlockNotFound(*cid)), + } + } + + async fn get_bytes<'a>( + &'a self, + cid: &'a Cid, + ) -> StoreResult>> { + self.layout.retrieve(cid, self.clone()).await + } + + async fn get_raw_block(&self, cid: &Cid) -> StoreResult { + let blocks = self.blocks.read().await; + match blocks.get(cid) { + Some((_, bytes)) => match cid.codec().try_into()? { + Codec::Raw => Ok(bytes.clone()), + codec => Err(StoreError::UnexpectedBlockCodec(Codec::Raw, codec)), + }, + None => Err(StoreError::BlockNotFound(*cid)), + } + } + + #[inline] + async fn has(&self, cid: &Cid) -> bool { + let blocks = self.blocks.read().await; + blocks.contains_key(cid) + } + + fn get_supported_codecs(&self) -> HashSet { + let mut codecs = HashSet::new(); + codecs.insert(Codec::DagCbor); + codecs.insert(Codec::Raw); + codecs + } + + #[inline] + fn get_node_block_max_size(&self) -> Option { + self.chunker.chunk_max_size() + } + + #[inline] + fn get_raw_block_max_size(&self) -> Option { + self.chunker.chunk_max_size() + } + + async fn is_empty(&self) -> StoreResult { + Ok(self.blocks.read().await.is_empty()) + } + + async fn get_size(&self) -> StoreResult { + Ok(self.blocks.read().await.len() as u64) + } +} + +impl IpldStoreSeekable for MemoryStore +where + C: Chunker + Clone + Send + Sync, + L: LayoutSeekable + Clone + Send + Sync, +{ + async fn get_seekable_bytes<'a>( + &'a self, + cid: &'a Cid, + ) -> StoreResult>> { + self.layout.retrieve_seekable(cid, self.clone()).await + } +} + +impl Default for MemoryStore { + fn default() -> Self { + MemoryStore { + blocks: Arc::new(RwLock::new(HashMap::new())), + chunker: FixedSizeChunker::default(), + layout: FlatLayout::default(), + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use tokio::io::AsyncReadExt; + + use super::*; + + #[tokio::test] + async fn test_memory_store_put_and_get() -> anyhow::Result<()> { + let store = MemoryStore::default(); + + //================== Raw ================== + + let data = vec![1, 2, 3, 4, 5]; + let cid = store.put_bytes(&data[..]).await?; + let mut res = store.get_bytes(&cid).await?; + + let mut buf = Vec::new(); + res.read_to_end(&mut buf).await?; + + assert_eq!(data, buf); + + //================= IPLD ================= + + let data = fixtures::Directory { + name: "root".to_string(), + entries: vec![ + utils::make_cid(Codec::Raw, &[1, 2, 3]), + utils::make_cid(Codec::Raw, &[4, 5, 6]), + ], + }; + + let cid = store.put_node(&data).await?; + let res = store.get_node::(&cid).await?; + + assert_eq!(res, data); + + Ok(()) + } +} + +#[cfg(test)] +mod fixtures { + use serde::Deserialize; + + use super::*; + + //-------------------------------------------------------------------------------------------------- + // Types + //-------------------------------------------------------------------------------------------------- + + #[derive(Clone, Serialize, Deserialize, Debug, PartialEq)] + pub(super) struct Directory { + pub(super) name: String, + pub(super) entries: Vec, + } + + //-------------------------------------------------------------------------------------------------- + // Trait Implementations + //-------------------------------------------------------------------------------------------------- + + impl IpldReferences for Directory { + fn get_references<'a>(&'a self) -> Box + Send + 'a> { + Box::new(self.entries.iter()) + } + } +} diff --git a/monoutils-store/lib/implementations/stores/mod.rs b/monoutils-store/lib/implementations/stores/mod.rs new file mode 100644 index 0000000..19a2e48 --- /dev/null +++ b/monoutils-store/lib/implementations/stores/mod.rs @@ -0,0 +1,11 @@ +mod dualstore; +mod memstore; +mod plcstore; + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +pub use dualstore::*; +pub use memstore::*; +pub use plcstore::*; diff --git a/monoutils-store/lib/implementations/stores/plcstore.rs b/monoutils-store/lib/implementations/stores/plcstore.rs new file mode 100644 index 0000000..008da8d --- /dev/null +++ b/monoutils-store/lib/implementations/stores/plcstore.rs @@ -0,0 +1,80 @@ +use std::{collections::HashSet, pin::Pin}; + +use bytes::Bytes; +use libipld::Cid; +use serde::Serialize; +use tokio::io::AsyncRead; + +use crate::{Codec, IpldReferences, IpldStore, StoreResult}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A placeholder store that does nothing. It is meant to be used as a placeholder for a store that +/// is not yet implemented. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PlaceholderStore; + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl IpldStore for PlaceholderStore { + async fn put_node(&self, _: &T) -> StoreResult + where + T: Serialize + IpldReferences, + { + unimplemented!("placeholder") + } + + async fn put_bytes<'a>(&'a self, _: impl AsyncRead + Send + 'a) -> StoreResult { + unimplemented!("placeholder") + } + + async fn put_raw_block(&self, _: impl Into) -> StoreResult { + unimplemented!("placeholder") + } + + async fn get_node(&self, _: &Cid) -> StoreResult + where + D: serde::de::DeserializeOwned, + { + unimplemented!("placeholder") + } + + async fn get_bytes<'a>( + &'a self, + _: &'a Cid, + ) -> StoreResult>> { + unimplemented!("placeholder") + } + + async fn get_raw_block(&self, _: &Cid) -> StoreResult { + unimplemented!("placeholder") + } + + async fn has(&self, _: &Cid) -> bool { + unimplemented!("placeholder") + } + + fn get_supported_codecs(&self) -> HashSet { + unimplemented!("placeholder") + } + + fn get_node_block_max_size(&self) -> Option { + unimplemented!("placeholder") + } + + fn get_raw_block_max_size(&self) -> Option { + unimplemented!("placeholder") + } + + async fn is_empty(&self) -> StoreResult { + unimplemented!("placeholder") + } + + async fn get_size(&self) -> StoreResult { + unimplemented!("placeholder") + } +} diff --git a/monoutils-store/lib/layout.rs b/monoutils-store/lib/layout.rs new file mode 100644 index 0000000..881e408 --- /dev/null +++ b/monoutils-store/lib/layout.rs @@ -0,0 +1,45 @@ +use std::pin::Pin; + +use bytes::Bytes; +use futures::{stream::BoxStream, Future}; +use libipld::Cid; +use tokio::io::AsyncRead; + +use super::{IpldStore, SeekableReader, StoreResult}; + +//-------------------------------------------------------------------------------------------------- +// Traits +//-------------------------------------------------------------------------------------------------- + +/// A layout strategy for organizing a stream of chunks into a graph of blocks. +pub trait Layout { + /// Organizes a stream of chunks into a graph of blocks storing them as either raw blocks or + /// IPLD node blocks. + /// + /// Method returns a stream of `Cid`s of the blocks that were created and the last `Cid` is + /// always the root of the graph. + fn organize<'a>( + &self, + stream: BoxStream<'a, StoreResult>, + store: impl IpldStore + Send + Sync + 'a, + ) -> impl Future>>> + Send; + + /// Retrieves the underlying byte chunks associated with a given `Cid`. + /// + /// This traverses the graph of blocks to reconstruct the original byte stream. + fn retrieve<'a>( + &self, + cid: &Cid, + store: impl IpldStore + Send + Sync + 'a, + ) -> impl Future>>> + Send; +} + +/// A trait that extends the `Layout` trait to allow for seeking. +pub trait LayoutSeekable: Layout { + /// Retrieves the underlying byte chunks associated with a given `Cid` as a seekable reader. + fn retrieve_seekable<'a>( + &self, + cid: &'a Cid, + store: impl IpldStore + Send + Sync + 'a, + ) -> impl Future>>> + Send; +} diff --git a/monoutils-store/lib/lib.rs b/monoutils-store/lib/lib.rs new file mode 100644 index 0000000..98a9930 --- /dev/null +++ b/monoutils-store/lib/lib.rs @@ -0,0 +1,41 @@ +//! Content-addressed storage (CAS) module. +//! +//! This module provides utilities for working with content-addressed storage (CAS). CAS is a +//! storage paradigm where data is addressed by its content, rather than by its location. + +#![warn(missing_docs)] +#![allow(clippy::module_inception)] + +mod chunker; +mod error; +mod implementations; +mod layout; +mod merkle; +mod references; +mod seekable; +mod storable; +mod store; +pub(crate) mod utils; + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- + +pub use chunker::*; +pub use error::*; +pub use implementations::*; +pub use layout::*; +pub use merkle::*; +pub use references::*; +pub use seekable::*; +pub use storable::*; +pub use store::*; + +//-------------------------------------------------------------------------------------------------- +// Re-Exports +//-------------------------------------------------------------------------------------------------- + +/// Re-exports of the `libipld` crate. +pub mod ipld { + pub use libipld::{cid, codec, multihash}; +} diff --git a/monoutils-store/lib/merkle.rs b/monoutils-store/lib/merkle.rs new file mode 100644 index 0000000..6ac2456 --- /dev/null +++ b/monoutils-store/lib/merkle.rs @@ -0,0 +1,66 @@ +use libipld::Cid; +use serde::{Deserialize, Serialize}; + +use super::IpldReferences; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A `MerkleNode` is a simple data structure for constructing a Directed Acyclic Graph (DAG) out of +/// multiple leaf data. +/// +/// It is a non-leaf data structure containing references (CIDs) to its direct dependencies which +/// can be either leaf or non-leaf data structures. +/// +/// This data structure is usually used internally by `IpldStore`s to store chunked data in a way that +/// preserves the original order of the data. See [`MemoryStore`] for an example of how this is +/// used. +/// +/// ## Important +/// +/// The serialized form of this data structure is typically expected to fit in a single node block. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct MerkleNode { + /// The size in bytes of the data this node represents. + pub size: usize, + + // /// The parent of the merkle node. + // pub parent: Option, + /// The direct children of this node, represented as a vector of tuples. + /// Each tuple contains: + /// - `Cid`: The content identifier (CID) of the child node. + /// - `usize`: The size in bytes of the data referenced by the CID. + pub children: Vec<(Cid, usize)>, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl MerkleNode { + /// Create a new `MerkleNode` with the given size and dependencies. + pub fn new(children: impl IntoIterator) -> Self { + let mut size = 0; + let mut deps = Vec::new(); + for (cid, byte_size) in children { + size += byte_size; + deps.push((cid, byte_size)); + } + + Self { + size, + children: deps, + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl IpldReferences for MerkleNode { + fn get_references<'a>(&'a self) -> Box + Send + 'a> { + Box::new(self.children.iter().map(|(cid, _)| cid)) + } +} diff --git a/monoutils-store/lib/references.rs b/monoutils-store/lib/references.rs new file mode 100644 index 0000000..fe55c90 --- /dev/null +++ b/monoutils-store/lib/references.rs @@ -0,0 +1,97 @@ +use std::iter; + +use bytes::Bytes; +use libipld::Cid; + +//-------------------------------------------------------------------------------------------------- +// Traits +//-------------------------------------------------------------------------------------------------- + +/// A trait for types that can hold [CID][cid] references to some data. +/// +/// [cid]: https://docs.ipfs.tech/concepts/content-addressing/ +pub trait IpldReferences { + /// Returns all the direct CID references the type has to other data. + fn get_references<'a>(&'a self) -> Box + Send + 'a>; +} + +//-------------------------------------------------------------------------------------------------- +// Macros +//-------------------------------------------------------------------------------------------------- + +macro_rules! impl_ipld_references { + (($($name:ident),+)) => { + impl<$($name),+> IpldReferences for ($($name,)+) + where + $($name: IpldReferences,)* + { + fn get_references<'a>(&'a self) -> Box + Send + 'a> { + #[allow(non_snake_case)] + let ($($name,)+) = self; + Box::new( + Vec::new().into_iter() + $(.chain($name.get_references()))+ + ) + } + } + }; + ($type:ty) => { + impl IpldReferences for $type { + fn get_references<'a>(&'a self) -> Box + Send + 'a> { + Box::new(std::iter::empty()) + } + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +// Nothing +impl_ipld_references!(()); + +// Scalars +impl_ipld_references!(bool); +impl_ipld_references!(u8); +impl_ipld_references!(u16); +impl_ipld_references!(u32); +impl_ipld_references!(u64); +impl_ipld_references!(u128); +impl_ipld_references!(usize); +impl_ipld_references!(i8); +impl_ipld_references!(i16); +impl_ipld_references!(i32); +impl_ipld_references!(i64); +impl_ipld_references!(i128); +impl_ipld_references!(isize); +impl_ipld_references!(f32); +impl_ipld_references!(f64); + +// Containers +impl_ipld_references!(Vec); +impl_ipld_references!(&[u8]); +impl_ipld_references!(Bytes); +impl_ipld_references!(String); +impl_ipld_references!(&str); + +// Tuples +impl_ipld_references!((A, B)); +impl_ipld_references!((A, B, C)); +impl_ipld_references!((A, B, C, D)); +impl_ipld_references!((A, B, C, D, E)); +impl_ipld_references!((A, B, C, D, E, F)); +impl_ipld_references!((A, B, C, D, E, F, G)); +impl_ipld_references!((A, B, C, D, E, F, G, H)); + +impl IpldReferences for Option +where + T: IpldReferences, +{ + fn get_references<'a>(&'a self) -> Box + Send + 'a> { + match self { + Some(value) => Box::new(value.get_references()), + None => Box::new(iter::empty()), + } + } +} diff --git a/monoutils-store/lib/seekable.rs b/monoutils-store/lib/seekable.rs new file mode 100644 index 0000000..d3c691c --- /dev/null +++ b/monoutils-store/lib/seekable.rs @@ -0,0 +1,14 @@ +use tokio::io::{AsyncRead, AsyncSeek}; + +//-------------------------------------------------------------------------------------------------- +// Traits +//-------------------------------------------------------------------------------------------------- + +/// A trait that extends the `AsyncRead` and `AsyncSeek` traits to allow for seeking. +pub trait SeekableReader: AsyncRead + AsyncSeek {} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl SeekableReader for T where T: AsyncRead + AsyncSeek {} diff --git a/monoutils-store/lib/storable.rs b/monoutils-store/lib/storable.rs new file mode 100644 index 0000000..ebb347d --- /dev/null +++ b/monoutils-store/lib/storable.rs @@ -0,0 +1,20 @@ +use futures::Future; +use libipld::Cid; + +use super::{IpldStore, StoreResult}; + +//-------------------------------------------------------------------------------------------------- +// Traits +//-------------------------------------------------------------------------------------------------- + +/// A trait that all types that need to be stored in an IPLD store must implement. +pub trait Storable: Sized +where + S: IpldStore, +{ + /// Stores the type in the IPLD store and returns the Cid. + fn store(&self) -> impl Future>; + + /// Loads the type from the IPLD store. + fn load(cid: &Cid, store: S) -> impl Future>; +} diff --git a/monoutils-store/lib/store.rs b/monoutils-store/lib/store.rs new file mode 100644 index 0000000..6bd1f53 --- /dev/null +++ b/monoutils-store/lib/store.rs @@ -0,0 +1,197 @@ +use std::{collections::HashSet, future::Future, pin::Pin}; + +use bytes::Bytes; +use libipld::Cid; +use serde::{de::DeserializeOwned, Serialize}; +use tokio::io::{AsyncRead, AsyncReadExt}; + +use super::{IpldReferences, SeekableReader, StoreError, StoreResult}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// The different codecs supported by the IPLD store. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Codec { + /// Raw bytes. + Raw, + + /// DAG-CBOR codec. + DagCbor, + + /// DAG-JSON codec. + DagJson, + + /// DAG-PB codec. + DagPb, +} + +//-------------------------------------------------------------------------------------------------- +// Traits: IpldStore, IpldStoreSeekable, IpldStoreExt +//-------------------------------------------------------------------------------------------------- + +/// `IpldStore` is a content-addressable store for [`IPLD` (InterPlanetary Linked Data)][ipld] that +/// emphasizes the structured nature of the data it stores. +/// +/// It can store raw bytes of data and structured data stored as IPLD. Stored data can be fetched +/// by their [`CID`s (Content Identifier)][cid] which is represents the fingerprint of the data. +/// +/// ## Important +/// +/// It is highly recommended to implement `Clone` with inexpensive cloning semantics. This is because +/// `IpldStore`s are usually passed around a lot and cloned to be used in different parts of the +/// application. +/// +/// An implementation is responsible for how it stores, encodes and chunks data into blocks. +/// +/// [cid]: https://docs.ipfs.tech/concepts/content-addressing/ +/// [ipld]: https://ipld.io/ +/// +// TODO: Add support for deleting blocks with `derefence` method. +// TODO: Add support for specifying hash type. +pub trait IpldStore: Clone { + /// Saves an IPLD serializable object to the store and returns the `Cid` to it. + /// + /// ## Errors + /// + /// If the serialized data is too large, `StoreError::NodeBlockTooLarge` error is returned. + fn put_node(&self, data: &T) -> impl Future> + Send + where + T: Serialize + IpldReferences + Sync; + + /// Takes a reader of raw bytes, saves it to the store and returns the `Cid` to it. + /// + /// This method allows the store to chunk large amounts of data into smaller blocks to fit the + /// storage medium and it may also involve creation of merkle nodes to represent the chunks. + /// + /// ## Errors + /// + /// If the bytes are too large, `StoreError::RawBlockTooLarge` error is returned. + fn put_bytes<'a>( + &'a self, + reader: impl AsyncRead + Send + Sync + 'a, + ) -> impl Future> + 'a; + + /// Tries to save `bytes` as a single block to the store. Unlike `put_bytes`, this method does + /// not chunk the data and does not create intermediate merkle nodes. + /// + /// ## Errors + /// + /// If the bytes are too large, `StoreError::RawBlockTooLarge` error is returned. + fn put_raw_block( + &self, + bytes: impl Into + Send, + ) -> impl Future> + Send; + + /// Gets a type stored as an IPLD data from the store by its `Cid`. + /// + /// ## Errors + /// + /// If the block is not found, `StoreError::BlockNotFound` error is returned. + fn get_node(&self, cid: &Cid) -> impl Future> + Send + where + D: DeserializeOwned + Send; + + /// Gets a reader for the underlying bytes associated with the given `Cid`. + /// + /// ## Errors + /// + /// If the block is not found, `StoreError::BlockNotFound` error is returned. + fn get_bytes<'a>( + &'a self, + cid: &'a Cid, + ) -> impl Future>>> + 'a; + + /// Retrieves raw bytes of a single block from the store by its `Cid`. + /// + /// Unlike `get_stream`, this method does not expect chunked data and does not have to retrieve + /// intermediate merkle nodes. + /// + /// ## Errors + /// + /// If the block is not found, `StoreError::BlockNotFound` error is returned. + fn get_raw_block(&self, cid: &Cid) -> impl Future> + Send + Sync; + + /// Checks if the store has a block with the given `Cid`. + fn has(&self, cid: &Cid) -> impl Future; + + /// Returns the codecs supported by the store. + fn get_supported_codecs(&self) -> HashSet; + + /// Returns the allowed maximum block size for IPLD and merkle nodes. + /// If there is no limit, `None` is returned. + fn get_node_block_max_size(&self) -> Option; + + /// Returns the allowed maximum block size for raw bytes. If there is no limit, `None` is returned. + fn get_raw_block_max_size(&self) -> Option; + + /// Checks if the store is empty. + fn is_empty(&self) -> impl Future>; + + /// Returns the number of blocks in the store. + fn get_size(&self) -> impl Future>; + + // /// Attempts to remove a node and its dependencies from the store. + // /// + // /// Returns the number of nodes and blocks removed. + // fn remove(&self, cid: &Cid) -> impl Future>; +} + +/// Helper extension to the `IpldStore` trait. +pub trait IpldStoreExt: IpldStore { + /// Reads all the bytes associated with the given `Cid` into a single `Bytes` type. + fn read_all(&self, cid: &Cid) -> impl Future> { + async { + let mut reader = self.get_bytes(cid).await?; + let mut bytes = Vec::new(); + + reader + .read_to_end(&mut bytes) + .await + .map_err(StoreError::custom)?; + + Ok(Bytes::from(bytes)) + } + } +} + +/// `IpldStoreSeekable` is a trait that extends the `IpldStore` trait to allow for seeking. +pub trait IpldStoreSeekable: IpldStore { + /// Gets a seekable reader for the underlying bytes associated with the given `Cid`. + fn get_seekable_bytes<'a>( + &'a self, + cid: &'a Cid, + ) -> impl Future>>>; +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl TryFrom for Codec { + type Error = StoreError; + + fn try_from(value: u64) -> Result { + match value { + 0x55 => Ok(Codec::Raw), + 0x71 => Ok(Codec::DagCbor), + 0x0129 => Ok(Codec::DagJson), + 0x70 => Ok(Codec::DagPb), + _ => Err(StoreError::UnsupportedCodec(value)), + } + } +} + +impl From for u64 { + fn from(codec: Codec) -> Self { + match codec { + Codec::Raw => 0x55, + Codec::DagCbor => 0x71, + Codec::DagJson => 0x0129, + Codec::DagPb => 0x70, + } + } +} + +impl IpldStoreExt for T where T: IpldStore {} diff --git a/monoutils-store/lib/utils.rs b/monoutils-store/lib/utils.rs new file mode 100644 index 0000000..ce3a14f --- /dev/null +++ b/monoutils-store/lib/utils.rs @@ -0,0 +1,23 @@ +use libipld::{ + multihash::{Code, MultihashDigest}, + Cid, +}; + +use super::Codec; + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Hashes data with [Blake3-256][blake] and returns a new [`Cid`] to it. +/// +/// [blake]: https://en.wikipedia.org/wiki/BLAKE_(hash_function) +pub(crate) fn make_cid(codec: Codec, data: &[u8]) -> Cid { + let digest = match codec { + Codec::Raw => Code::Blake3_256.digest(data), + Codec::DagCbor => Code::Blake3_256.digest(data), + Codec::DagJson => Code::Blake3_256.digest(data), + Codec::DagPb => Code::Blake3_256.digest(data), + }; + Cid::new_v1(codec.into(), digest) +} diff --git a/monoutils-ucan/Cargo.toml b/monoutils-ucan/Cargo.toml new file mode 100644 index 0000000..4442412 --- /dev/null +++ b/monoutils-ucan/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "monoutils-ucan" +version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "monoutils_ucan" +path = "lib/lib.rs" + +[dependencies] diff --git a/monoutils-ucan/lib/lib.rs b/monoutils-ucan/lib/lib.rs new file mode 100644 index 0000000..9d8b570 --- /dev/null +++ b/monoutils-ucan/lib/lib.rs @@ -0,0 +1,8 @@ +//! `monobase` is a distributed, decentralized, secure multi-model database. + +#![warn(missing_docs)] +#![allow(clippy::module_inception)] + +//-------------------------------------------------------------------------------------------------- +// Exports +//-------------------------------------------------------------------------------------------------- diff --git a/monovue/Cargo.toml b/monovue/Cargo.toml index 7b77cde..1ec3966 100644 --- a/monovue/Cargo.toml +++ b/monovue/Cargo.toml @@ -1,8 +1,10 @@ [package] name = "monovue" -version = "0.1.0" -edition = "2021" +version.workspace = true license.workspace = true +repository.workspace = true +authors.workspace = true +edition.workspace = true [lib] name = "monovue"