Open indexes outside of the main MultiIndex lock

Author: lalinsky

src/Index.zig

@@ -35,7 +35,6 @@ const metrics = @import("metrics.zig");
 const Self = @This();
 
 const Options = struct {
-    create: bool = false,
     min_segment_size: usize = 500_000,
     max_segment_size: usize = 500_000_000,
 };
@@ -151,13 +150,13 @@ fn loadSegment(self: *Self, info: SegmentInfo) !FileSegmentNode {
     return node;
 }
 
-fn loadSegments(self: *Self) !u64 {
+fn loadSegments(self: *Self, create: bool) !u64 {
     self.segments_lock.lock();
     defer self.segments_lock.unlock();
 
     const segment_ids = filefmt.readManifestFile(self.dir, self.allocator) catch |err| {
         if (err == error.FileNotFound) {
-            if (self.options.create) {
+            if (create) {
                 try self.updateManifestFile(self.file_segments.segments.value);
                 return 0;
             }
@@ -362,8 +361,8 @@ fn stopMemorySegmentMergeThread(self: *Self) void {
     self.memory_segment_merge_thread = null;
 }
 
-pub fn open(self: *Self) !void {
-    const last_commit_id = try self.loadSegments();
+pub fn open(self: *Self, create: bool) !void {
+    const last_commit_id = try self.loadSegments(create);
 
     // start these threads after loading file segments, but before replaying oplog to memory segments
     try self.startFileSegmentMergeThread();

src/MultiIndex.zig

@@ -10,6 +10,8 @@ pub const IndexRef = struct {
     index: Index,
     references: usize = 0,
     last_used_at: i64 = std.math.minInt(i64),
+    is_open: bool = false,
+    lock: std.Thread.Mutex = .{},
 };
 
 lock: std.Thread.Mutex = .{},
@@ -71,7 +73,7 @@ pub fn releaseIndex(self: *Self, index_data: *IndexRef) void {
     index_data.last_used_at = std.time.timestamp();
 }
 
-pub fn acquireIndex(self: *Self, name: []const u8, create: bool) !*IndexRef {
+pub fn acquireIndex(self: *Self, name: []const u8) !*IndexRef {
     if (!isValidName(name)) {
         return error.InvalidIndexName;
     }
@@ -91,30 +93,42 @@ pub fn acquireIndex(self: *Self, name: []const u8, create: bool) !*IndexRef {
     result.key_ptr.* = try self.allocator.dupe(u8, name);
     errdefer self.allocator.free(result.key_ptr.*);
 
-    result.value_ptr.index = try Index.init(self.allocator, self.dir, name, .{ .create = create });
+    result.value_ptr.index = try Index.init(self.allocator, self.dir, name, .{});
     errdefer result.value_ptr.index.deinit();
 
-    try result.value_ptr.index.open();
-
+    result.value_ptr.is_open = false;
     result.value_ptr.references = 1;
     result.value_ptr.last_used_at = std.time.timestamp();
     return result.value_ptr;
 }
 
+fn ensureOpen(index_ref: *IndexRef, create: bool) !void {
+    index_ref.lock.lock();
+    defer index_ref.lock.unlock();
+
+    if (index_ref.is_open) return;
+
+    try index_ref.index.open(create);
+    index_ref.is_open = true;
+}
+
 pub fn getIndex(self: *Self, name: []const u8) !*IndexRef {
     if (!isValidName(name)) {
         return error.InvalidIndexName;
     }
 
-    return try self.acquireIndex(name, false);
+    const index_ref = try self.acquireIndex(name);
+    try ensureOpen(index_ref, false);
+    return index_ref;
 }
 
 pub fn createIndex(self: *Self, name: []const u8) !void {
     if (!isValidName(name)) {
         return error.InvalidIndexName;
     }
 
-    const index_ref = try self.acquireIndex(name, true);
+    const index_ref = try self.acquireIndex(name);
+    try ensureOpen(index_ref, true);
     defer self.releaseIndex(index_ref);
 }
 

src/index_tests.zig

@@ -22,18 +22,18 @@ test "index does not exist" {
     var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{});
     defer index.deinit();
 
-    const result = index.open();
+    const result = index.open(false);
     try std.testing.expectError(error.IndexNotFound, result);
 }
 
 test "index create, update and search" {
     var tmp_dir = std.testing.tmpDir(.{});
     defer tmp_dir.cleanup();
 
-    var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{ .create = true });
+    var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{});
     defer index.deinit();
 
-    try index.open();
+    try index.open(true);
 
     var hashes: [100]u32 = undefined;
 
@@ -69,10 +69,10 @@ test "index create, update, reopen and search" {
     var hashes: [100]u32 = undefined;
 
     {
-        var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{ .create = true });
+        var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{});
         defer index.deinit();
 
-        try index.open();
+        try index.open(true);
 
         try index.update(&[_]Change{.{ .insert = .{
             .id = 1,
@@ -81,10 +81,10 @@ test "index create, update, reopen and search" {
     }
 
     {
-        var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{ .create = false });
+        var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{});
         defer index.deinit();
 
-        try index.open();
+        try index.open(false);
 
         var results = try index.search(generateRandomHashes(&hashes, 1), std.testing.allocator, .{});
         defer results.deinit();
@@ -105,10 +105,10 @@ test "index many updates" {
     var hashes: [100]u32 = undefined;
 
     {
-        var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{ .create = true });
+        var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{});
         defer index.deinit();
 
-        try index.open();
+        try index.open(true);
 
         for (0..100) |i| {
             try index.update(&[_]Change{.{ .insert = .{
@@ -118,10 +118,10 @@ test "index many updates" {
         }
     }
 
-    var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{ .create = false });
+    var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{});
     defer index.deinit();
 
-    try index.open();
+    try index.open(false);
 
     {
         var results = try index.search(generateRandomHashes(&hashes, 0), std.testing.allocator, .{});
@@ -146,10 +146,10 @@ test "index, multiple fingerprints with the same hashes" {
     var tmp_dir = std.testing.tmpDir(.{});
     defer tmp_dir.cleanup();
 
-    var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{ .create = true });
+    var index = try Index.init(std.testing.allocator, tmp_dir.dir, "idx", .{});
     defer index.deinit();
 
-    try index.open();
+    try index.open(true);
 
     var hashes: [100]u32 = undefined;