Fix tests

Author: lalinsky

.github/workflows/build.yml

@@ -24,8 +24,6 @@ jobs:
           python-version: '3.13'
           cache: 'pip'
 
-      - run: sudo apt install -y valgrind
-
       - run: zig build
       - run: zig build unit-tests --summary all
 

src/Index.zig

@@ -405,8 +405,6 @@ pub fn update(self: *Self, changes: []const Change) !void {
 }
 
 pub fn updateInternal(self: *Self, changes: []const Change, commit_id: ?u64) !void {
-    // log.debug("update with {} changes", .{changes.len});
-
     var target = try MemorySegmentList.createSegment(self.allocator, .{});
     defer MemorySegmentList.destroySegment(self.allocator, &target);
 

src/main.zig

@@ -1,4 +1,5 @@
 const std = @import("std");
+const log = std.log.scoped(.main);
 const zul = @import("zul");
 
 const MultiIndex = @import("MultiIndex.zig");
@@ -51,8 +52,12 @@ pub fn main() !void {
         }
     }
 
-    const dir_path = args.get("dir") orelse ".";
+    const dir_path_relative = args.get("dir") orelse "/tmp/fpindex";
+    const dir_path = try std.fs.cwd().realpathAlloc(allocator, dir_path_relative);
+    defer allocator.free(dir_path);
+
     const dir = try std.fs.cwd().makeOpenPath(dir_path, .{ .iterate = true });
+    log.info("using directory {s}", .{dir_path});
 
     const address = args.get("address") orelse "127.0.0.1";
 
@@ -66,6 +71,7 @@ pub fn main() !void {
     if (threads == 0) {
         threads = @intCast(try std.Thread.getCpuCount());
     }
+    log.info("using {} threads", .{threads});
 
     try metrics.initializeMetrics(.{ .prefix = "aindex_" });
 

src/server.zig

@@ -72,8 +72,9 @@ pub fn run(allocator: std.mem.Allocator, indexes: *MultiIndex, address: []const
     var router = server.router();
 
     // Monitoring API
-    router.get("/_ping", handlePing);
     router.get("/_metrics", handleMetrics);
+    router.get("/_ping", handlePing);
+    router.get("/:index/_ping", handlePingIndex);
 
     // Search API
     router.post("/:index/_search", handleSearch);
@@ -277,11 +278,6 @@ fn handleUpdate(ctx: *Context, req: *httpz.Request, res: *httpz.Response) !void
     return writeResponse(EmptyResponse{}, req, res);
 }
 
-fn handleHeadIndex(ctx: *Context, req: *httpz.Request, res: *httpz.Response) !void {
-    const index_ref = try getIndex(ctx, req, res, false) orelse return;
-    defer releaseIndex(ctx, index_ref);
-}
-
 const Attributes = struct {
     attributes: std.AutoHashMapUnmanaged(u64, u64),
 
@@ -348,6 +344,18 @@ fn handleDeleteIndex(ctx: *Context, req: *httpz.Request, res: *httpz.Response) !
     return writeResponse(EmptyResponse{}, req, res);
 }
 
+fn handleHeadIndex(ctx: *Context, req: *httpz.Request, res: *httpz.Response) !void {
+    const index_ref = try getIndex(ctx, req, res, false) orelse return;
+    defer releaseIndex(ctx, index_ref);
+}
+
+fn handlePingIndex(ctx: *Context, req: *httpz.Request, res: *httpz.Response) !void {
+    const index_ref = try getIndex(ctx, req, res, false) orelse return;
+    defer releaseIndex(ctx, index_ref);
+
+    try handlePing(ctx, req, res);
+}
+
 fn handlePing(ctx: *Context, req: *httpz.Request, res: *httpz.Response) !void {
     _ = ctx;
     _ = req;

tests/conftest.py

@@ -2,7 +2,6 @@
 import pytest
 import subprocess
 import time
-import socket
 from urllib.parse import urljoin
 
 
@@ -16,7 +15,6 @@ def __init__(self, base_dir, port):
 
     def start(self):
         command = [
-            'valgrind',
             'zig-out/bin/fpindex',
             '--dir', str(self.data_dir),
             '--port', str(self.port),
@@ -26,19 +24,44 @@ def start(self):
             command,
             stdin=subprocess.DEVNULL,
             stdout=subprocess.DEVNULL,
-            stderr=self.log_file.open('w'),
+            stderr=self.log_file.open('a'),
         )
+        self.wait_for_ready()
 
-    def stop(self):
+    def stop(self, kill=False):
         if self.process is not None:
             if self.process.returncode is None:
-                self.process.terminate()
+                if kill:
+                    self.process.kill()
+                else:
+                    self.process.terminate()
                 try:
                     self.process.wait(timeout=1.0)
                 except subprocess.TimeoutExpired:
                     self.process.kill()
                     self.process.wait()
 
+    def restart(self, kill=True):
+        self.stop(kill=kill)
+        self.start()
+
+    def wait_for_ready(self, index_name=None, timeout=10.0):
+        deadline = time.time() + timeout
+        while True:
+            if index_name:
+                url = f'http://localhost:{self.port}/{index_name}/_ping'
+            else:
+                url = f'http://localhost:{self.port}/_ping'
+            try:
+                with requests.get(url) as res:
+                    res.raise_for_status()
+            except Exception:
+                if time.time() > deadline:
+                    raise TimeoutError()
+                time.sleep(timeout / 100.0)
+            else:
+                break
+
     def error_log(self):
         for line in self.log_file.read_text().splitlines():
             yield line
@@ -48,10 +71,12 @@ def error_log(self):
 def server(tmp_path_factory):
     srv = ServerManager(base_dir=tmp_path_factory.mktemp('srv'), port=14502)
     srv.start()
-    yield srv
-    srv.stop()
-    for line in srv.error_log():
-        print(line)
+    try:
+        yield srv
+    finally:
+        srv.stop()
+        for line in srv.error_log():
+            print(line)
 
 
 index_no = 1
@@ -96,22 +121,8 @@ def session():
         yield session
 
 
-def check_port(port_no):
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        return sock.connect_ex(('127.0.0.1', port_no)) == 0
-
-
-def wait_for_ready(port, timeout):
-    deadline = time.time() + timeout
-    while not check_port(port):
-        if time.time() > deadline:
-            raise TimeoutError()
-        time.sleep(timeout / 100.0)
-
-
 @pytest.fixture
 def client(session, server):
-    wait_for_ready(server.port, 1)
     return Client(session, f'http://localhost:{server.port}')
 
 

tests/test_fingerprint_api.py

@@ -124,3 +124,56 @@ def test_delete(client, index_name, create_index):
     assert json.loads(req.content) == {
         'results': []
     }
+
+
+def test_persistence_after_soft_restart(server, client, index_name, create_index):
+    # insert fingerprint
+    for i in range(100):
+        body = {
+            'changes': [
+                {'insert': {'id': 1, 'hashes': [100+i, 200+i, 300+i]}}
+            ],
+        }
+        with client.post(f'/{index_name}/_update', json=body) as req:
+            assert req.status_code == 200, req.content
+            assert json.loads(req.content) == {}
+
+    server.restart()
+    server.wait_for_ready(index_name, timeout=10.0)
+
+    # verify we can find it
+    req = client.post(f'/{index_name}/_search', json={
+        'query': [199, 299, 399]
+    })
+    assert req.status_code == 200, req.content
+    assert json.loads(req.content) == {
+        'results': [{'id': 1, 'score': 3}]
+    }
+
+
+def test_persistence_after_hard_restart(server, client, index_name, create_index):
+    # insert fingerprint
+    for i in range(100):
+        body = {
+            'changes': [
+                {'insert': {'id': 1, 'hashes': [100+i, 200+i, 300+i]}}
+            ],
+        }
+        with client.post(f'/{index_name}/_update', json=body) as req:
+            assert req.status_code == 200, req.content
+            assert json.loads(req.content) == {}
+    assert req.status_code == 200, req.content
+    assert json.loads(req.content) == {}
+
+    server.restart(kill=True)
+    server.wait_for_ready(index_name, timeout=10.0)
+
+    # verify we can find it
+    req = client.post(f'/{index_name}/_search', json={
+        'query': [199, 299, 399]
+    })
+    assert req.status_code == 200, req.content
+    assert json.loads(req.content) == {
+        'results': [{'id': 1, 'score': 3}]
+    }
+