Merge branch '106-hnsw-annotation' into 'main'

Add property index annotation for HNSW

See merge request objectbox/objectbox-dart!78

Author: greenrobot-team

dev-doc/updating-c-library.md

@@ -9,17 +9,17 @@ For Dart Native and unit tests ([install.sh](../install.sh)),
 for the binding update script (see below) and
 for Flutter (`flutter_libs` and `sync_flutter_libs` plugins) on Linux and Windows:  
 ```
-./tool/set-c-version.sh 0.21.0
+./tool/set-c-version.sh 4.0.0
 ```
 
 For the Flutter plugins on Android ([view releases](https://github.com/objectbox/objectbox-java/releases)):
 ```
-./tool/set-android-version.sh 3.8.0
+./tool/set-android-version.sh 4.0.0
 ```
 
 For the Flutter plugins on iOS/macOS ([view releases](https://github.com/objectbox/objectbox-swift/releases))
 ```
-./tool/set-swift-version.sh 1.9.2
+./tool/set-swift-version.sh 2.0.0
 ```
 
 For each, add an entry (see previous releases) to the [CHANGELOG](../objectbox/CHANGELOG.md).

flutter_libs/android/build.gradle

@@ -52,6 +52,6 @@ android {
         // ObjectBox Android library that includes an ObjectBox C library version compatible with
         // the C API binding of the ObjectBox Dart package.
         // https://central.sonatype.com/search?q=g:io.objectbox%20objectbox-android
-        implementation "io.objectbox:objectbox-android:3.8.0"
+        implementation "io.objectbox:objectbox-android:4.0.0"
     }
 }

flutter_libs/ios/objectbox_flutter_libs.podspec

@@ -18,7 +18,7 @@ Pod::Spec.new do |s|
   s.source_files     = 'Classes/**/*'
 
   s.dependency 'Flutter'
-  s.dependency 'ObjectBox', '1.9.2'
+  s.dependency 'ObjectBox', '2.0.0'
 
   # Flutter.framework does not contain a i386 slice.
   s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' }

flutter_libs/linux/CMakeLists.txt

@@ -44,7 +44,7 @@ target_link_libraries(${PLUGIN_NAME} PRIVATE PkgConfig::GTK)
 # ----------------------------------------------------------------------
 # Download and add objectbox-c prebuilt library.
 
-set(OBJECTBOX_VERSION 0.21.0)
+set(OBJECTBOX_VERSION 4.0.0)
 
 set(OBJECTBOX_ARCH ${CMAKE_SYSTEM_PROCESSOR})
 if (${OBJECTBOX_ARCH} MATCHES "x86_64")

flutter_libs/macos/objectbox_flutter_libs.podspec

@@ -18,7 +18,7 @@ Pod::Spec.new do |s|
   s.source_files     = 'Classes/**/*'
 
   s.dependency 'FlutterMacOS'
-  s.dependency 'ObjectBox', '1.9.2'
+  s.dependency 'ObjectBox', '2.0.0'
 
   s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES' }
   s.swift_version = '5.3'

flutter_libs/windows/CMakeLists.txt

@@ -50,7 +50,7 @@ set(objectbox_flutter_libs_bundled_libraries
 # ----------------------------------------------------------------------
 # Download and add objectbox-c prebuilt library.
 
-set(OBJECTBOX_VERSION 0.21.0)
+set(OBJECTBOX_VERSION 4.0.0)
 
 set(OBJECTBOX_ARCH ${CMAKE_SYSTEM_PROCESSOR})
 if (${OBJECTBOX_ARCH} MATCHES "AMD64")

generator/lib/src/code_builder.dart

@@ -211,6 +211,7 @@ class CodeBuilder extends Builder {
     propInModel.flags = prop.flags;
     propInModel.dartFieldType = prop.dartFieldType;
     propInModel.relationTarget = prop.relationTarget;
+    propInModel.hnswParams = prop.hnswParams;
 
     if (!prop.hasIndexFlag()) {
       propInModel.removeIndex();

generator/lib/src/code_chunks.dart

@@ -1,6 +1,7 @@
 import 'package:build/build.dart';
 import 'package:collection/collection.dart' show IterableExtension;
 import 'package:objectbox/internal.dart';
+import 'package:objectbox/objectbox.dart';
 import 'package:pubspec_parse/pubspec_parse.dart';
 import 'package:source_gen/source_gen.dart' show InvalidGenerationSourceError;
 
@@ -149,6 +150,10 @@ class CodeChunks {
         property.relationTarget!.isNotEmpty) {
       additionalArgs += ", relationTarget: '${property.relationTarget!}'";
     }
+    if (property.hnswParams != null) {
+      additionalArgs +=
+          ", hnswParams: ${property.hnswParams!.toCodeString(obxInt)}";
+    }
     return '''
     $obxInt.ModelProperty(
       id: ${createIdUid(property.id)},
@@ -714,6 +719,8 @@ class CodeChunks {
       if (prop.isRelation) {
         propCode +=
             '$obx.QueryRelationToOne<${entity.name}, ${prop.relationTarget}>';
+      } else if (prop.hnswParams != null) {
+        propCode += '$obx.QueryHnswProperty<${entity.name}>';
       } else {
         propCode += '$obx.Query${fieldType}Property<${entity.name}>';
       }

generator/lib/src/entity_resolver.dart

@@ -27,6 +27,7 @@ class EntityResolver extends Builder {
   final _uniqueChecker = const TypeChecker.fromRuntime(Unique);
   final _indexChecker = const TypeChecker.fromRuntime(Index);
   final _backlinkChecker = const TypeChecker.fromRuntime(Backlink);
+  final _hnswChecker = const TypeChecker.fromRuntime(HnswIndex);
 
   @override
   FutureOr<void> build(BuildStep buildStep) async {
@@ -67,6 +68,7 @@ class EntityResolver extends Builder {
         null,
         uidRequest: !entityUid.isNull && entityUid.intValue == 0);
 
+    // Sync: check if enabled
     if (_syncChecker.hasAnnotationOfExact(classElement)) {
       entity.flags |= OBXEntityFlags.SYNC_ENABLED;
     }
@@ -207,6 +209,20 @@ class EntityResolver extends Builder {
         processAnnotationIndexUnique(
             f, annotated, fieldType, classElement, prop);
 
+        // Vector database: check for any HNSW index params
+        _hnswChecker.runIfMatches(annotated, (annotation) {
+          // Note: using other index annotations on FloatVector currently
+          // errors, so no need to integrate with regular index processing.
+          if (fieldType != OBXPropertyType.FloatVector) {
+            throw InvalidGenerationSourceError(
+                "'${classElement.name}.${f.name}': @HnswIndex is only supported for float vector properties.",
+                element: f);
+          }
+          // Create an index
+          prop.flags |= OBXPropertyFlags.INDEXED;
+          _readHnswIndexParams(annotation, prop);
+        });
+
         // for code generation
         prop.dartFieldType =
             f.type.element!.name! + (isNullable(f.type) ? '?' : '');
@@ -383,8 +399,9 @@ class EntityResolver extends Builder {
 
     // If available use index type from annotation.
     if (indexAnnotation != null && !indexAnnotation.isNull) {
-      final enumValItem = enumValueItem(indexAnnotation.getField('type')!);
-      if (enumValItem != null) indexType = IndexType.values[enumValItem];
+      final typeIndex =
+          _enumValueIndex(indexAnnotation.getField('type')!, "Index.type");
+      if (typeIndex != null) indexType = IndexType.values[typeIndex];
     }
 
     // Fall back to index type based on property type.
@@ -408,10 +425,11 @@ class EntityResolver extends Builder {
     if (uniqueAnnotation != null && !uniqueAnnotation.isNull) {
       prop.flags |= OBXPropertyFlags.UNIQUE;
       // Determine unique conflict resolution.
-      final onConflictVal =
-          enumValueItem(uniqueAnnotation.getField('onConflict')!);
-      if (onConflictVal != null &&
-          ConflictStrategy.values[onConflictVal] == ConflictStrategy.replace) {
+      final onConflictIndex = _enumValueIndex(
+          uniqueAnnotation.getField('onConflict')!, "Unique.onConflict");
+      if (onConflictIndex != null &&
+          ConflictStrategy.values[onConflictIndex] ==
+              ConflictStrategy.replace) {
         prop.flags |= OBXPropertyFlags.UNIQUE_ON_CONFLICT_REPLACE;
       }
     }
@@ -459,28 +477,22 @@ class EntityResolver extends Builder {
     }
   }
 
-  int? enumValueItem(DartObject typeField) {
-    if (!typeField.isNull) {
-      final enumValues = (typeField.type as InterfaceType)
-          .element
-          .fields
-          .where((f) => f.isEnumConstant)
-          .toList();
-
-      // Find the index of the matching enum constant.
-      for (var i = 0; i < enumValues.length; i++) {
-        if (enumValues[i].computeConstantValue() == typeField) {
-          return i;
-        }
-      }
+  /// If not null, returns the index of the enum value.
+  int? _enumValueIndex(DartObject enumState, String fieldName) {
+    if (enumState.isNull) return null;
+    // All enum classes implement the Enum interface
+    // which has the index property.
+    final index = enumState.getField("index")?.toIntValue();
+    if (index == null) {
+      throw ArgumentError.value(enumState, fieldName,
+          "Dart object state does not appear to represent an enum");
     }
-
-    return null;
+    return index;
   }
 
   // find out @Property(type:) field value - its an enum PropertyType
   int? propertyTypeFromAnnotation(DartObject typeField) {
-    final item = enumValueItem(typeField);
+    final item = _enumValueIndex(typeField, "Property.type");
     return item == null
         ? null
         : propertyTypeToOBXPropertyType(PropertyType.values[item]);
@@ -522,6 +534,28 @@ class EntityResolver extends Builder {
       return info.toString();
     }).toList(growable: false);
   }
+
+  void _readHnswIndexParams(DartObject annotation, ModelProperty property) {
+    final distanceTypeIndex = _enumValueIndex(
+        annotation.getField('distanceType')!, "HnswIndex.distanceType");
+    final distanceType = distanceTypeIndex != null
+        ? VectorDistanceType.values[distanceTypeIndex]
+        : null;
+
+    final hnswRestored = HnswIndex(
+        dimensions: annotation.getField('dimensions')!.toIntValue()!,
+        neighborsPerNode: annotation.getField('neighborsPerNode')!.toIntValue(),
+        indexingSearchCount:
+            annotation.getField('indexingSearchCount')!.toIntValue(),
+        flags: _HnswFlagsState.fromState(annotation.getField('flags')!),
+        distanceType: distanceType,
+        reparationBacklinkProbability: annotation
+            .getField('reparationBacklinkProbability')!
+            .toDoubleValue(),
+        vectorCacheHintSizeKB:
+            annotation.getField('vectorCacheHintSizeKB')!.toIntValue());
+    property.hnswParams = ModelHnswParams.fromAnnotation(hnswRestored);
+  }
 }
 
 extension _TypeCheckerExtensions on TypeChecker {
@@ -530,3 +564,18 @@ extension _TypeCheckerExtensions on TypeChecker {
     if (annotations.isNotEmpty) fn(annotations.first);
   }
 }
+
+extension _HnswFlagsState on HnswFlags {
+  static HnswFlags? fromState(DartObject state) {
+    if (state.isNull) return null;
+    return HnswFlags(
+        debugLogs: state.getField('debugLogs')!.toBoolValue() ?? false,
+        debugLogsDetailed:
+            state.getField('debugLogsDetailed')!.toBoolValue() ?? false,
+        vectorCacheSimdPaddingOff:
+            state.getField('vectorCacheSimdPaddingOff')!.toBoolValue() ?? false,
+        reparationLimitCandidates:
+            state.getField('reparationLimitCandidates')!.toBoolValue() ??
+                false);
+  }
+}

generator/test/code_builder_test.dart

@@ -242,6 +242,116 @@ void main() {
       List<String>? tStrings;
       ''');
     });
+
+    test('HNSW annotation on unsupported type errors', () async {
+      final source = r'''
+      library example;     
+      import 'package:objectbox/objectbox.dart';
+      
+      @Entity()
+      class Example {
+        @Id()
+        int id = 0;
+        
+        @HnswIndex(dimensions: 3)
+        List<double>? coordinates;
+      }
+      ''';
+
+      final testEnv = GeneratorTestEnv();
+      await expectLater(
+          () async => await testEnv.run(source),
+          throwsA(isA<InvalidGenerationSourceError>().having(
+              (e) => e.message,
+              'message',
+              contains(
+                  "@HnswIndex is only supported for float vector properties."))));
+    });
+
+    test('HNSW annotation default', () async {
+      final source = r'''
+      library example;     
+      import 'package:objectbox/objectbox.dart';
+      
+      @Entity()
+      class Example {
+        @Id()
+        int id = 0;
+        
+        @Property(type: PropertyType.floatVector)
+        @HnswIndex(dimensions: 3)
+        List<double>? coordinates;
+      }
+      ''';
+
+      final testEnv = GeneratorTestEnv();
+      await testEnv.run(source);
+
+      // Assert final model created by generator
+      final vectorProperty = testEnv.model.entities[0].properties
+          .firstWhere((element) => element.name == "coordinates");
+      expect(vectorProperty.flags & OBXPropertyFlags.INDEXED != 0, true);
+      expect(vectorProperty.indexId, isNotNull);
+      expect(vectorProperty.hnswParams, isNotNull);
+      expect(vectorProperty.hnswParams!.dimensions, 3);
+      expect(vectorProperty.hnswParams!.neighborsPerNode, isNull);
+      expect(vectorProperty.hnswParams!.indexingSearchCount, isNull);
+      expect(vectorProperty.hnswParams!.flags, isNull);
+      expect(vectorProperty.hnswParams!.distanceType, isNull);
+      expect(vectorProperty.hnswParams!.reparationBacklinkProbability, isNull);
+      expect(vectorProperty.hnswParams!.vectorCacheHintSizeKB, isNull);
+    });
+
+    test('HNSW annotation with all properties', () async {
+      final source = r'''
+      library example;     
+      import 'package:objectbox/objectbox.dart';
+      
+      @Entity()
+      class Example {
+        @Id()
+        int id = 0;
+        
+        @Property(type: PropertyType.floatVector)
+        @HnswIndex(
+            dimensions: 3,
+            neighborsPerNode: 30,
+            indexingSearchCount: 100,
+            flags: HnswFlags(
+                debugLogs: true,
+                debugLogsDetailed: true,
+                vectorCacheSimdPaddingOff: true,
+                reparationLimitCandidates: true),
+            distanceType: VectorDistanceType.euclidean,
+            reparationBacklinkProbability: 0.95,
+            vectorCacheHintSizeKB: 2097152)
+        List<double>? coordinates;
+      }
+      ''';
+
+      final testEnv = GeneratorTestEnv();
+      await testEnv.run(source);
+
+      // Assert final model created by generator
+      final vectorProperty = testEnv.model.entities[0].properties
+          .firstWhere((element) => element.name == "coordinates");
+      expect(vectorProperty.flags & OBXPropertyFlags.INDEXED != 0, true);
+      expect(vectorProperty.indexId, isNotNull);
+      expect(vectorProperty.hnswParams, isNotNull);
+      expect(vectorProperty.hnswParams!.dimensions, 3);
+      expect(vectorProperty.hnswParams!.neighborsPerNode, 30);
+      expect(vectorProperty.hnswParams!.indexingSearchCount, 100);
+      final flags = vectorProperty.hnswParams!.flags;
+      expect(flags, isNotNull);
+      expect(flags! & OBXHnswFlags.DebugLogs != 0, true);
+      expect(flags & OBXHnswFlags.DebugLogsDetailed != 0, true);
+      expect(flags & OBXHnswFlags.VectorCacheSimdPaddingOff != 0, true);
+      expect(flags & OBXHnswFlags.ReparationLimitCandidates != 0, true);
+      expect(vectorProperty.hnswParams!.distanceType,
+          OBXVectorDistanceType.Euclidean);
+      expect(vectorProperty.hnswParams!.reparationBacklinkProbability, 0.95);
+      expect(vectorProperty.hnswParams!.vectorCacheHintSizeKB, 2097152);
+    });
   });
 }
 

install.sh

@@ -5,7 +5,7 @@ set -eu
 # It's important that the generated dart bindings and the c-api library version match. Dart won't error on C function
 # signature mismatch, leading to obscure memory bugs.
 # For how to upgrade the version see dev-doc/updating-c-library.md
-cLibVersion=0.21.0
+cLibVersion=4.0.0
 os=$(uname)
 cLibArgs="$*"