update

Utils/Package.swift

@@ -60,7 +60,8 @@ let package = Package(
             dependencies: [
                 "Utils",
                 .product(name: "Testing", package: "swift-testing"),
-            ]
+            ],
+            resources: [.copy("Resources")]
         ),
     ],
     swiftLanguageVersions: [.version("6")]

Utils/Sources/erasure-coding/bindings.h

@@ -8,6 +8,8 @@
 #define POINT_SIZE 2
 #define SUBSHARD_POINTS 6
 
+typedef uint16_t ChunkIndex;
+
 
 
 #define MAX_CHUNKS 16384
@@ -38,31 +40,39 @@
 #define SUBSHARD_SIZE (POINT_SIZE * SUBSHARD_POINTS)
 
 /**
- * The index of an erasure chunk.
+ * Subshard uses some temp memory, so these should be used multiple time instead of allocating.
  */
-typedef struct ChunkIndex ChunkIndex;
+typedef struct SubShardDecoder SubShardDecoder;
 
 /**
- * Result of the reconstruct.
+ * Subshard uses some temp memory, so these should be used multiple time instead of allocating.
  */
-typedef struct ReconstructResult ReconstructResult;
+typedef struct SubShardEncoder SubShardEncoder;
 
-/**
- * Fix size segment of a larger data.
- * Data is padded when unaligned with
- * the segment size.
- */
-typedef struct Segment Segment;
+typedef struct CSegment {
+  uint8_t *data;
+  uint32_t index;
+} CSegment;
 
-/**
- * Subshard uses some temp memory, so these should be used multiple time instead of allocating.
- */
-typedef struct SubShardDecoder SubShardDecoder;
+typedef struct SegmentTuple {
+  uint8_t index;
+  struct CSegment segment;
+} SegmentTuple;
 
 /**
- * Subshard uses some temp memory, so these should be used multiple time instead of allocating.
+ * Result of the reconstruct.
  */
-typedef struct SubShardEncoder SubShardEncoder;
+typedef struct ReconstructResult {
+  struct SegmentTuple *segments;
+  uintptr_t num_segments;
+  uintptr_t num_decodes;
+} ReconstructResult;
+
+typedef struct SubShardTuple {
+  uint8_t seg_index;
+  ChunkIndex chunk_index;
+  uint8_t shard[12];
+} SubShardTuple;
 
 /**
  * Subshard (points in sequential orders).
@@ -86,7 +96,7 @@ void subshard_encoder_free(struct SubShardEncoder *encoder);
  * out_len is N * TOTAL_SHARDS
  */
 void subshard_encoder_construct(struct SubShardEncoder *encoder,
-                                const struct Segment *segments,
+                                const struct CSegment *segments,
                                 uintptr_t num_segments,
                                 bool *success,
                                 uint8_t (***out_chunks)[12],
@@ -102,6 +112,10 @@ struct SubShardDecoder *subshard_decoder_new(void);
  */
 void subshard_decoder_free(struct SubShardDecoder *decoder);
 
-uintptr_t reconstruct_result_get_num_decodes(const struct ReconstructResult *result);
-
-void reconstruct_result_free(struct ReconstructResult *result);
+/**
+ * Reconstructs data from a list of subshards.
+ */
+struct ReconstructResult *subshard_decoder_reconstruct(struct SubShardDecoder *decoder,
+                                                       const struct SubShardTuple *subshards,
+                                                       uintptr_t num_subshards,
+                                                       bool *success);

Utils/Sources/erasure-coding/cbindgen.toml

@@ -5,6 +5,8 @@ after_includes = """
 #define POINT_SIZE 2
 #define SUBSHARD_POINTS 6
 
+typedef uint16_t ChunkIndex;
+
 """
 
 autogen_warning = "/* Warning, this file is auto generated by cbindgen. Don't modify this manually. */"
@@ -16,4 +18,5 @@ include = ["erasure-coding"]
 extra_bindings = ["erasure-coding"]
 
 [export]
-include = ["Segment", "SubShard", "POINT_SIZE", "SUBSHARD_SIZE", "ChunkIndex"]
+include = ["SubShard", "POINT_SIZE", "SUBSHARD_SIZE"]
+exclude = ["ChunkIndex"]

Utils/Sources/erasure-coding/src/ffi.rs

@@ -1,6 +1,42 @@
-use erasure_coding::{ChunkIndex, Segment, SubShardDecoder, SubShardEncoder, TOTAL_SHARDS};
+use erasure_coding::{
+    ChunkIndex, Segment, SubShardDecoder, SubShardEncoder, SEGMENT_SIZE, TOTAL_SHARDS,
+};
 use std::slice;
 
+#[repr(C)]
+#[derive(Clone, Copy, Debug)]
+pub struct CSegment {
+    data: *mut u8, // Pointer to the data, length is SEGMENT_SIZE
+    index: u32,
+}
+
+impl From<Segment> for CSegment {
+    fn from(segment: Segment) -> Self {
+        let mut vec_data = Vec::from(*segment.data);
+
+        let c_segment = CSegment {
+            data: vec_data.as_mut_ptr(),
+            index: segment.index,
+        };
+
+        // prevent Rust from freeing the Vec while CSegment is in use
+        std::mem::forget(vec_data);
+
+        c_segment
+    }
+}
+
+impl From<CSegment> for Segment {
+    fn from(c_segment: CSegment) -> Self {
+        let vec_data = unsafe { Vec::from_raw_parts(c_segment.data, SEGMENT_SIZE, SEGMENT_SIZE) };
+
+        Segment {
+            data: Box::new(vec_data.try_into().unwrap()),
+            index: c_segment.index,
+        }
+    }
+}
+
 /// Initializes a new SubShardEncoder.
 #[no_mangle]
 pub extern "C" fn subshard_encoder_new() -> *mut SubShardEncoder {
@@ -22,7 +58,7 @@ pub extern "C" fn subshard_encoder_free(encoder: *mut SubShardEncoder) {
 #[no_mangle]
 pub extern "C" fn subshard_encoder_construct(
     encoder: *mut SubShardEncoder,
-    segments: *const Segment,
+    segments: *const CSegment,
     num_segments: usize,
     success: *mut bool,
     out_chunks: *mut *mut *mut [u8; 12],
@@ -34,9 +70,15 @@ pub extern "C" fn subshard_encoder_construct(
     }
 
     let encoder = unsafe { &mut *encoder };
-    let segments = unsafe { slice::from_raw_parts(segments, num_segments) };
-
-    match encoder.construct_chunks(segments) {
+    let segments_vec: Vec<Segment> = unsafe {
+        slice::from_raw_parts(segments, num_segments)
+            .iter()
+            .map(|segment| Segment::from(*segment))
+            .collect()
+    };
+    let r_segments: &[Segment] = &segments_vec;
+
+    match encoder.construct_chunks(r_segments) {
         Ok(result) => {
             let total_chunks = result.len() * TOTAL_SHARDS;
             let mut chunk_ptrs: Vec<*mut [u8; 12]> = Vec::with_capacity(total_chunks);
@@ -77,47 +119,69 @@ pub extern "C" fn subshard_decoder_free(decoder: *mut SubShardDecoder) {
     }
 }
 
+#[repr(C)]
+pub struct SegmentTuple {
+    pub index: u8,
+    pub segment: CSegment,
+}
+
 /// Result of the reconstruct.
 #[repr(C)]
 pub struct ReconstructResult {
-    pub segments: *mut (u8, Segment),
+    pub segments: *mut SegmentTuple,
     pub num_segments: usize,
     pub num_decodes: usize,
 }
 
-#[no_mangle]
-pub extern "C" fn reconstruct_result_get_segments(
-    result: *const ReconstructResult,
-) -> *const (u8, Segment) {
-    if result.is_null() {
-        return std::ptr::null();
-    }
-    unsafe { (*result).segments }
-}
+// #[no_mangle]
+// pub extern "C" fn reconstruct_result_get_segments(
+//     result: *const ReconstructResult,
+// ) -> *const SegmentTuple {
+//     if result.is_null() {
+//         return std::ptr::null();
+//     }
+//     unsafe { (*result).segments }
+// }
+
+// #[no_mangle]
+// pub extern "C" fn reconstruct_result_get_num_segments(result: *const ReconstructResult) -> usize {
+//     if result.is_null() {
+//         return 0;
+//     }
+//     unsafe { (*result).num_segments }
+// }
+
+// #[no_mangle]
+// pub extern "C" fn reconstruct_result_get_num_decodes(result: *const ReconstructResult) -> usize {
+//     if result.is_null() {
+//         return 0;
+//     }
+//     unsafe { (*result).num_decodes }
+// }
+
+// #[no_mangle]
+// pub extern "C" fn reconstruct_result_free(result: *mut ReconstructResult) {
+//     if !result.is_null() {
+//         unsafe {
+//             let boxed_result = Box::from_raw(result);
+//             drop(Box::from_raw(boxed_result.segments));
+//             drop(boxed_result);
+//         }
+//     }
+// }
 
-#[no_mangle]
-pub extern "C" fn reconstruct_result_get_num_decodes(result: *const ReconstructResult) -> usize {
-    if result.is_null() {
-        return 0;
-    }
-    unsafe { (*result).num_decodes }
-}
-
-#[no_mangle]
-pub extern "C" fn reconstruct_result_free(result: *mut ReconstructResult) {
-    if !result.is_null() {
-        unsafe {
-            let boxed_result = Box::from_raw(result);
-            drop(Box::from_raw(boxed_result.segments));
-        }
-    }
+#[repr(C)]
+pub struct SubShardTuple {
+    pub seg_index: u8,
+    pub chunk_index: ChunkIndex,
+    pub shard: [u8; 12],
 }
 
 /// Reconstructs data from a list of subshards.
 #[no_mangle]
 pub extern "C" fn subshard_decoder_reconstruct(
     decoder: *mut SubShardDecoder,
-    subshards: *const (u8, ChunkIndex, [u8; 12]),
+    subshards: *const SubShardTuple,
     num_subshards: usize,
     success: *mut bool,
 ) -> *mut ReconstructResult {
@@ -131,12 +195,18 @@ pub extern "C" fn subshard_decoder_reconstruct(
 
     let cloned_subshards: Vec<(u8, ChunkIndex, &[u8; 12])> = subshards_slice
         .iter()
-        .map(|&(a, b, ref c)| (a, b, c))
+        .map(|t| (t.seg_index, t.chunk_index, &t.shard))
         .collect();
 
     match decoder.reconstruct(&mut cloned_subshards.iter().cloned()) {
         Ok((segments, num_decodes)) => {
-            let mut segments_vec: Vec<(u8, Segment)> = segments.into_iter().collect();
+            let mut segments_vec: Vec<SegmentTuple> = segments
+                .into_iter()
+                .map(|(index, segment)| SegmentTuple {
+                    index,
+                    segment: segment.into(),
+                })
+                .collect();
             let segments_len = segments_vec.len();
             let segments_ptr = segments_vec.as_mut_ptr();