Track min_document_id

lalinsky · lalinsky · commit a2813b9e3a6d · 2024-12-07T07:34:42.000+01:00
diff --git a/src/FileSegment.zig b/src/FileSegment.zig
@@ -22,6 +22,7 @@ dir: std.fs.Dir,
 info: SegmentInfo = .{},
 attributes: std.StringHashMapUnmanaged(u64) = .{},
 docs: std.AutoHashMapUnmanaged(u32, bool) = .{},
+min_doc_id: u32 = 0,
 max_doc_id: u32 = 0,
 index: std.ArrayListUnmanaged(u32) = .{},
 block_size: usize = 0,
diff --git a/src/IndexReader.zig b/src/IndexReader.zig
@@ -58,11 +58,26 @@ pub fn getDocInfo(self: *Self, doc_id: u32) !?DocInfo {
     return null;
 }
 
+pub fn getMinDocId(self: *Self) u32 {
+    var result: u32 = 0;
+    inline for (segment_lists) |n| {
+        const segments = @field(self, n);
+        const doc_id = segments.value.getMinDocId();
+        if (result == 0 or doc_id < result) {
+            result = doc_id;
+        }
+    }
+    return result;
+}
+
 pub fn getMaxDocId(self: *Self) u32 {
     var result: u32 = 0;
     inline for (segment_lists) |n| {
         const segments = @field(self, n);
-        result = @max(result, segments.value.getMaxDocId());
+        const doc_id = segments.value.getMaxDocId();
+        if (result == 0 or doc_id > result) {
+            result = doc_id;
+        }
     }
     return result;
 }
@@ -96,6 +111,7 @@ pub fn getAttributes(self: *Self, allocator: std.mem.Allocator) !std.StringHashM
     }
 
     // builtin attributes
+    try attributes.put(allocator, "min_document_id", self.getMinDocId());
     try attributes.put(allocator, "max_document_id", self.getMaxDocId());
 
     return attributes;
diff --git a/src/MemorySegment.zig b/src/MemorySegment.zig
@@ -21,6 +21,7 @@ allocator: std.mem.Allocator,
 info: SegmentInfo = .{},
 attributes: std.StringHashMapUnmanaged(u64) = .{},
 docs: std.AutoHashMapUnmanaged(u32, bool) = .{},
+min_doc_id: u32 = 0,
 max_doc_id: u32 = 0,
 items: std.ArrayListUnmanaged(Item) = .{},
 frozen: bool = false,
@@ -82,6 +83,7 @@ pub fn build(self: *Self, changes: []const Change) !void {
     try self.docs.ensureTotalCapacity(self.allocator, num_docs);
     try self.items.ensureTotalCapacity(self.allocator, num_items);
 
+    self.min_doc_id = 0;
     self.max_doc_id = 0;
     var i = changes.len;
     while (i > 0) {
@@ -96,7 +98,12 @@ pub fn build(self: *Self, changes: []const Change) !void {
                     for (op.hashes, 0..) |hash, j| {
                         items[j] = .{ .hash = hash, .id = op.id };
                     }
-                    self.max_doc_id = @max(self.max_doc_id, op.id);
+                    if (self.min_doc_id == 0 or op.id < self.min_doc_id) {
+                        self.min_doc_id = op.id;
+                    }
+                    if (self.max_doc_id == 0 or op.id > self.max_doc_id) {
+                        self.max_doc_id = op.id;
+                    }
                 }
             },
             .delete => |op| {
diff --git a/src/filefmt.zig b/src/filefmt.zig
@@ -437,9 +437,15 @@ pub fn readSegmentFile(dir: fs.Dir, info: SegmentInfo, segment: *FileSegment) !v
         segment.docs = docs.unmanaged.move();
 
         var iter = segment.docs.keyIterator();
+        segment.min_doc_id = 0;
         segment.max_doc_id = 0;
         while (iter.next()) |key_ptr| {
-            segment.max_doc_id = @max(segment.max_doc_id, key_ptr.*);
+            if (segment.min_doc_id == 0 or key_ptr.* < segment.min_doc_id) {
+                segment.min_doc_id = key_ptr.*;
+            }
+            if (segment.max_doc_id == 0 or key_ptr.* > segment.max_doc_id) {
+                segment.max_doc_id = key_ptr.*;
+            }
         }
     }
 
diff --git a/src/segment_list.zig b/src/segment_list.zig
@@ -110,10 +110,22 @@ pub fn SegmentList(Segment: type) type {
             }
         }
 
+        pub fn getMinDocId(self: Self) u32 {
+            var result: u32 = 0;
+            for (self.nodes.items) |node| {
+                if (result == 0 or node.value.min_doc_id < result) {
+                    result = node.value.min_doc_id;
+                }
+            }
+            return result;
+        }
+
         pub fn getMaxDocId(self: Self) u32 {
             var result: u32 = 0;
             for (self.nodes.items) |node| {
-                result = @max(result, node.value.max_doc_id);
+                if (result == 0 or node.value.max_doc_id > result) {
+                    result = node.value.max_doc_id;
+                }
             }
             return result;
         }
diff --git a/src/segment_merger.zig b/src/segment_merger.zig
@@ -9,6 +9,7 @@ pub const MergedSegmentInfo = struct {
     info: SegmentInfo = .{},
     attributes: std.StringHashMapUnmanaged(u64) = .{},
     docs: std.AutoHashMapUnmanaged(u32, bool) = .{},
+    min_doc_id: u32 = 0,
     max_doc_id: u32 = 0,
 
     pub fn deinit(self: *MergedSegmentInfo, allocator: std.mem.Allocator) void {
@@ -116,6 +117,7 @@ pub fn SegmentMerger(comptime Segment: type) type {
             }
 
             try self.segment.docs.ensureTotalCapacity(self.allocator, total_docs);
+            self.segment.min_doc_id = 0;
             self.segment.max_doc_id = 0;
             for (sources) |*source| {
                 const segment = source.reader.segment;
@@ -129,7 +131,12 @@ pub fn SegmentMerger(comptime Segment: type) type {
                     if (!self.collection.hasNewerVersion(doc_id, segment.info.version)) {
                         try self.segment.docs.put(self.allocator, doc_id, doc_status);
                         docs_added += 1;
-                        self.segment.max_doc_id = @max(self.segment.max_doc_id, doc_id);
+                        if (self.segment.min_doc_id == 0 or doc_id < self.segment.min_doc_id) {
+                            self.segment.min_doc_id = doc_id;
+                        }
+                        if (self.segment.max_doc_id == 0 or doc_id > self.segment.max_doc_id) {
+                            self.segment.max_doc_id = doc_id;
+                        }
                     } else {
                         try source.skip_docs.put(self.allocator, doc_id, {});
                     }
diff --git a/tests/test_index_api.py b/tests/test_index_api.py
@@ -34,9 +34,9 @@ def test_get_index(client, index_name, create_index, fmt):
     req = client.get(f'/{index_name}', headers=headers(fmt))
     assert req.status_code == 200, req.content
     if fmt == 'json':
-        expected = {'version': 1, 'segments': 1, 'attributes': {'foo': 1234, 'max_document_id': 1}}
+        expected = {'version': 1, 'segments': 1, 'attributes': {'foo': 1234, 'min_document_id': 1, 'max_document_id': 1}}
     else:
-        expected = {'v': 1, 's': 1, 'a': {'foo': 1234, 'max_document_id': 1}}
+        expected = {'v': 1, 's': 1, 'a': {'foo': 1234, 'min_document_id': 1, 'max_document_id': 1}}
     assert decode(fmt, req.content) == expected
 
 

Original file line number	Diff line number	Diff line change
`@@ -110,10 +110,22 @@ pub fn SegmentList(Segment: type) type {`
`110`	`110`	`}`
`111`	`111`	`}`
`112`	`112`
	`113`	`+ pub fn getMinDocId(self: Self) u32 {`
	`114`	`+ var result: u32 = 0;`
	`115`	`+ for (self.nodes.items) \|node\| {`
	`116`	`+ if (result == 0 or node.value.min_doc_id < result) {`
	`117`	`+ result = node.value.min_doc_id;`
	`118`	`+ }`
	`119`	`+ }`
	`120`	`+ return result;`
	`121`	`+ }`
	`122`	`+`
`113`	`123`	`pub fn getMaxDocId(self: Self) u32 {`
`114`	`124`	`var result: u32 = 0;`
`115`	`125`	`for (self.nodes.items) \|node\| {`
`116`		`- result = @max(result, node.value.max_doc_id);`
	`126`	`+ if (result == 0 or node.value.max_doc_id > result) {`
	`127`	`+ result = node.value.max_doc_id;`
	`128`	`+ }`
`117`	`129`	`}`
`118`	`130`	`return result;`
`119`	`131`	`}`