Add single fingerprint APIs

Author: lalinsky

src/Index.zig

@@ -9,6 +9,7 @@ const Change = @import("change.zig").Change;
 const SearchResult = @import("common.zig").SearchResult;
 const SearchResults = @import("common.zig").SearchResults;
 const SegmentInfo = @import("segment.zig").SegmentInfo;
+const DocInfo = @import("common.zig").DocInfo;
 
 const Oplog = @import("Oplog.zig");
 
@@ -475,6 +476,25 @@ pub fn search(self: *Self, hashes: []const u32, allocator: std.mem.Allocator, de
     return results;
 }
 
+pub fn getDocInfo(self: *Self, doc_id: u32) !?DocInfo {
+    var snapshot = self.acquireSegments();
+    defer self.releaseSegments(&snapshot); // FIXME this possibly deletes orphaned segments, do it in a separate thread
+
+    var result: ?DocInfo = null;
+    inline for (segment_lists) |n| {
+        const segments = @field(snapshot, n);
+        if (segments.value.getDocInfo(doc_id)) |res| {
+            result = res;
+        }
+    }
+    if (result) |res| {
+        if (!res.deleted) {
+            return res;
+        }
+    }
+    return null;
+}
+
 pub const IndexInfo = struct {
     version: u64,
     attributes: std.AutoHashMapUnmanaged(u64, u64),

src/common.zig

@@ -4,6 +4,11 @@ const testing = std.testing;
 const msgpack = @import("msgpack");
 const SegmentInfo = @import("segment.zig").SegmentInfo;
 
+pub const DocInfo = struct {
+    version: u64,
+    deleted: bool,
+};
+
 pub const KeepOrDelete = enum {
     keep,
     delete,

src/segment_list.zig

@@ -6,6 +6,7 @@ const SearchResults = @import("common.zig").SearchResults;
 const Change = @import("change.zig").Change;
 const KeepOrDelete = @import("common.zig").KeepOrDelete;
 const SegmentInfo = @import("segment.zig").SegmentInfo;
+const DocInfo = @import("common.zig").DocInfo;
 
 const Deadline = @import("utils/Deadline.zig");
 
@@ -100,6 +101,15 @@ pub fn SegmentList(Segment: type) type {
             }
         }
 
+        pub fn getDocInfo(self: Self, doc_id: u32) ?DocInfo {
+            var result: ?DocInfo = null;
+            for (self.nodes.items) |node| {
+                const active = node.value.docs.get(doc_id) orelse continue;
+                result = .{ .version = node.value.info.version, .deleted = !active };
+            }
+            return result;
+        }
+
         pub fn search(self: Self, hashes: []const u32, results: *SearchResults, deadline: Deadline) !void {
             for (self.nodes.items) |node| {
                 if (deadline.isExpired()) {

src/server.zig

@@ -82,6 +82,12 @@ pub fn run(allocator: std.mem.Allocator, indexes: *MultiIndex, address: []const
     // Bulk API
     router.post("/:index/_update", handleUpdate);
 
+    // Fingerprint API
+    router.head("/:index/:id", handleHeadFingerprint);
+    router.get("/:index/:id", handleGetFingerprint);
+    router.put("/:index/:id", handlePutFingerprint);
+    router.delete("/:index/:id", handleDeleteFingerprint);
+
     // Index API
     router.head("/:index", handleHeadIndex);
     router.get("/:index", handleGetIndex);
@@ -113,8 +119,34 @@ const SearchResultsJSON = struct {
     results: []SearchResultJSON,
 };
 
+fn getId(req: *httpz.Request, res: *httpz.Response, send_body: bool) !?u32 {
+    const id_str = req.param("id") orelse {
+        if (send_body) {
+            try writeErrorResponse(400, error.MissingId, req, res);
+        } else {
+            res.status = 400;
+        }
+        return null;
+    };
+    return std.fmt.parseInt(u32, id_str, 10) catch |err| {
+        if (send_body) {
+            try writeErrorResponse(400, err, req, res);
+        } else {
+            res.status = 400;
+        }
+        return null;
+    };
+}
+
 fn getIndex(ctx: *Context, req: *httpz.Request, res: *httpz.Response, send_body: bool) !?*IndexData {
-    const index_name = req.param("index") orelse return null;
+    const index_name = req.param("index") orelse {
+        if (send_body) {
+            try writeErrorResponse(400, error.MissingIndexName, req, res);
+        } else {
+            res.status = 400;
+        }
+        return null;
+    };
     const index = ctx.indexes.getIndex(index_name) catch |err| {
         if (err == error.IndexNotFound) {
             if (send_body) {
@@ -278,6 +310,83 @@ fn handleUpdate(ctx: *Context, req: *httpz.Request, res: *httpz.Response) !void
     return writeResponse(EmptyResponse{}, req, res);
 }
 
+fn handleHeadFingerprint(ctx: *Context, req: *httpz.Request, res: *httpz.Response) !void {
+    const index_ref = try getIndex(ctx, req, res, false) orelse return;
+    const index = &index_ref.index;
+    defer releaseIndex(ctx, index_ref);
+
+    const id = try getId(req, res, false) orelse return;
+    const info = try index.getDocInfo(id);
+
+    res.status = if (info == null) 404 else 200;
+}
+
+const GetFingerprintResponse = struct {
+    version: u64,
+
+    pub fn msgpackFormat() msgpack.StructFormat {
+        return .{ .as_map = .{ .key = .{ .field_name_prefix = 1 } } };
+    }
+};
+
+fn handleGetFingerprint(ctx: *Context, req: *httpz.Request, res: *httpz.Response) !void {
+    const index_ref = try getIndex(ctx, req, res, true) orelse return;
+    const index = &index_ref.index;
+    defer releaseIndex(ctx, index_ref);
+
+    const id = try getId(req, res, true) orelse return;
+    const info = try index.getDocInfo(id) orelse {
+        return writeErrorResponse(404, error.FingerprintNotFound, req, res);
+    };
+
+    return writeResponse(GetFingerprintResponse{ .version = info.version }, req, res);
+}
+
+const PutFingerprintRequest = struct {
+    hashes: []u32,
+
+    pub fn msgpackFormat() msgpack.StructFormat {
+        return .{ .as_map = .{ .key = .{ .field_name_prefix = 1 } } };
+    }
+};
+
+fn handlePutFingerprint(ctx: *Context, req: *httpz.Request, res: *httpz.Response) !void {
+    const body = try getRequestBody(PutFingerprintRequest, req, res) orelse return;
+
+    const index_ref = try getIndex(ctx, req, res, true) orelse return;
+    const index = &index_ref.index;
+    defer releaseIndex(ctx, index_ref);
+
+    const id = try getId(req, res, true) orelse return;
+    const change: Change = .{ .insert = .{
+        .id = id,
+        .hashes = body.hashes,
+    } };
+
+    metrics.update(1);
+
+    try index.update(&[_]Change{change});
+
+    return writeResponse(EmptyResponse{}, req, res);
+}
+
+fn handleDeleteFingerprint(ctx: *Context, req: *httpz.Request, res: *httpz.Response) !void {
+    const index_ref = try getIndex(ctx, req, res, true) orelse return;
+    const index = &index_ref.index;
+    defer releaseIndex(ctx, index_ref);
+
+    const id = try getId(req, res, true) orelse return;
+    const change: Change = .{ .delete = .{
+        .id = id,
+    } };
+
+    metrics.update(1);
+
+    try index.update(&[_]Change{change});
+
+    return writeResponse(EmptyResponse{}, req, res);
+}
+
 const Attributes = struct {
     attributes: std.AutoHashMapUnmanaged(u64, u64),
 

tests/test_fingerprint_api.py

@@ -20,6 +20,13 @@ def test_insert(client, index_name, create_index):
         'results': [{'id': 1, 'score': 3}]
     }
 
+    # verify we can get id
+    req = client.get(f'/{index_name}/1')
+    assert req.status_code == 200, req.content
+    assert json.loads(req.content) == {
+        'version': 1,
+    }
+
 
 def test_update_full(client, index_name, create_index):
     # insert fingerprint
@@ -58,6 +65,13 @@ def test_update_full(client, index_name, create_index):
         'results': [{'id': 1, 'score': 3}]
     }
 
+    # verify we can get id
+    req = client.get(f'/{index_name}/1')
+    assert req.status_code == 200, req.content
+    assert json.loads(req.content) == {
+        'version': 2,
+    }
+
 
 def test_update_partial(client, index_name, create_index):
     # insert fingerprint
@@ -96,6 +110,13 @@ def test_update_partial(client, index_name, create_index):
         'results': [{'id': 1, 'score': 3}]
     }
 
+    # verify we can get id
+    req = client.get(f'/{index_name}/1')
+    assert req.status_code == 200, req.content
+    assert json.loads(req.content) == {
+        'version': 2,
+    }
+
 
 def test_delete(client, index_name, create_index):
     # insert fingerprint
@@ -125,6 +146,13 @@ def test_delete(client, index_name, create_index):
         'results': []
     }
 
+    # verify we can get id
+    req = client.get(f'/{index_name}/1')
+    assert req.status_code == 404, req.content
+    assert json.loads(req.content) == {
+        'error': 'FingerprintNotFound',
+    }
+
 
 def test_persistence_after_soft_restart(server, client, index_name, create_index):
     # insert fingerprint
@@ -150,6 +178,13 @@ def test_persistence_after_soft_restart(server, client, index_name, create_index
         'results': [{'id': 1, 'score': 3}]
     }
 
+    # verify we can get id
+    req = client.get(f'/{index_name}/1')
+    assert req.status_code == 200, req.content
+    assert json.loads(req.content) == {
+        'version': 100,
+    }
+
 
 def test_persistence_after_hard_restart(server, client, index_name, create_index):
     # insert fingerprint
@@ -177,3 +212,9 @@ def test_persistence_after_hard_restart(server, client, index_name, create_index
         'results': [{'id': 1, 'score': 3}]
     }
 
+    # verify we can get id
+    req = client.get(f'/{index_name}/1')
+    assert req.status_code == 200, req.content
+    assert json.loads(req.content) == {
+        'version': 100,
+    }