feat: Add TS vector field filters support (#12376)

# Implementation Details
- Added support for 5 operators: `contains`, `containsAny`,
`containsAll`, `matches`, and `fuzzy`
- Works on any field of type `TS_VECTOR`
- Added PostgreSQL `pg_trgm` extension for fuzzy search functionality.
The extension provides the `similarity()` function needed for text
similarity searches.
- Not implemented in GraphQL

## Tradeoffs & Decisions
1. **Fuzzy Search Performance**: Using `pg_trgm` for fuzzy search is
more accurate but slower than simple text matching. We might want to add
a similarity threshold parameter in the future to control the tradeoff
between accuracy and performance.

2. **Operator Naming**: Chose `contains`/`containsAny`/`containsAll` to
be consistent with existing filter operators, though they might be less
intuitive than `search`/`searchAny`/`searchAll`.

## Demo


https://github.com/user-attachments/assets/790fc3ed-a188-4b49-864f-996a37481d99

---------

Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
Co-authored-by: Félix Malfait <felix@twenty.com>

Author: abdulrahmancodes

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-filter/graphql-query-filter-field.parser.ts

@@ -64,12 +64,12 @@ export class GraphqlQueryFilterFieldParser {
       );
     }
 
-    const { sql, params } = computeWhereConditionParts(
+    const { sql, params } = computeWhereConditionParts({
       operator,
       objectNameSingular,
       key,
       value,
-    );
+    });
 
     if (isFirst) {
       queryBuilder.where(sql, params);
@@ -124,12 +124,12 @@ export class GraphqlQueryFilterFieldParser {
         );
       }
 
-      const { sql, params } = computeWhereConditionParts(
+      const { sql, params } = computeWhereConditionParts({
         operator,
         objectNameSingular,
-        fullFieldName,
+        key: fullFieldName,
         value,
-      );
+      });
 
       if (isFirst && index === 0) {
         queryBuilder.where(sql, params);

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/compute-where-condition-parts.ts

@@ -10,13 +10,18 @@ type WhereConditionParts = {
   params: ObjectLiteral;
 };
 
-export const computeWhereConditionParts = (
-  operator: string,
-  objectNameSingular: string,
-  key: string,
+export const computeWhereConditionParts = ({
+  operator,
+  objectNameSingular,
+  key,
+  value,
+}: {
+  operator: string;
+  objectNameSingular: string;
+  key: string;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  value: any,
-): WhereConditionParts => {
+  value: any;
+}): WhereConditionParts => {
   const uuid = Math.random().toString(36).slice(2, 7);
 
   switch (operator) {
@@ -90,6 +95,23 @@ export const computeWhereConditionParts = (
         sql: `"${objectNameSingular}"."${key}" @> ARRAY[:...${key}${uuid}]`,
         params: { [`${key}${uuid}`]: value },
       };
+    case 'search': {
+      const tsQuery = value
+        .split(/\s+/)
+        .map((term: string) => `${term}:*`)
+        .join(' & ');
+
+      return {
+        sql: `(
+          "${objectNameSingular}"."${key}" @@ to_tsquery('simple', :${key}${uuid}Ts) OR
+          "${objectNameSingular}"."${key}"::text ILIKE :${key}${uuid}Like
+        )`,
+        params: {
+          [`${key}${uuid}Ts`]: tsQuery,
+          [`${key}${uuid}Like`]: `%${value}%`,
+        },
+      };
+    }
     case 'notContains':
       return {
         sql: `NOT ("${objectNameSingular}"."${key}"::text[] && ARRAY[:...${key}${uuid}]::text[])`,
@@ -105,7 +127,6 @@ export const computeWhereConditionParts = (
         sql: `EXISTS (SELECT 1 FROM unnest("${objectNameSingular}"."${key}") AS elem WHERE elem ILIKE :${key}${uuid})`,
         params: { [`${key}${uuid}`]: value },
       };
-
     default:
       throw new GraphqlQueryRunnerException(
         `Operator "${operator}" is not supported`,

packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/input/ts-vector-filter.input-type.ts

@@ -0,0 +1,8 @@
+import { GraphQLInputObjectType, GraphQLString } from 'graphql';
+
+export const TSVectorFilterType = new GraphQLInputObjectType({
+  name: 'TSVectorFilter',
+  fields: {
+    search: { type: GraphQLString },
+  },
+});

packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/index.ts

@@ -5,13 +5,17 @@ import { DateScalarType } from './date.scalar';
 import { PositionScalarType } from './position.scalar';
 import { RawJSONScalar } from './raw-json.scalar';
 import { TimeScalarType } from './time.scalar';
+import { TSVectorScalarType } from './ts-vector.scalar';
 import { UUIDScalarType } from './uuid.scalar';
 
 export * from './big-float.scalar';
 export * from './big-int.scalar';
 export * from './cursor.scalar';
 export * from './date.scalar';
+export * from './position.scalar';
+export * from './raw-json.scalar';
 export * from './time.scalar';
+export * from './ts-vector.scalar';
 export * from './uuid.scalar';
 
 export const scalars = [
@@ -23,4 +27,5 @@ export const scalars = [
   CursorScalarType,
   PositionScalarType,
   RawJSONScalar,
+  TSVectorScalarType,
 ];

packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/ts-vector.scalar.ts

@@ -0,0 +1,19 @@
+import { GraphQLScalarType } from 'graphql';
+
+export const TSVectorScalarType = new GraphQLScalarType({
+  name: 'TSVector',
+  description: 'A custom scalar type for PostgreSQL tsvector fields',
+  serialize(value: string): string {
+    return value;
+  },
+  parseValue(value: string): string {
+    return value;
+  },
+  parseLiteral(ast): string | null {
+    if (ast.kind === 'StringValue') {
+      return ast.value;
+    }
+
+    return null;
+  },
+});

packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/services/type-mapper.service.ts

@@ -30,9 +30,11 @@ import {
 import { MultiSelectFilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/multi-select-filter.input-type';
 import { RichTextV2FilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/rich-text.input-type';
 import { SelectFilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/select-filter.input-type';
+import { TSVectorFilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/ts-vector-filter.input-type';
 import { UUIDFilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/uuid-filter.input-type';
 import {
   BigFloatScalarType,
+  TSVectorScalarType,
   UUIDScalarType,
 } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
 import { PositionScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/position.scalar';
@@ -83,7 +85,7 @@ export class TypeMapperService {
         StringArrayScalarType as unknown as GraphQLScalarType,
       ],
       [FieldMetadataType.RICH_TEXT, GraphQLString],
-      [FieldMetadataType.TS_VECTOR, GraphQLString],
+      [FieldMetadataType.TS_VECTOR, TSVectorScalarType],
     ]);
 
     return typeScalarMapping.get(fieldMetadataType);
@@ -122,7 +124,7 @@ export class TypeMapperService {
       [FieldMetadataType.ARRAY, ArrayFilterType],
       [FieldMetadataType.MULTI_SELECT, MultiSelectFilterType],
       [FieldMetadataType.SELECT, SelectFilterType],
-      [FieldMetadataType.TS_VECTOR, StringFilterType], // TODO: Add TSVectorFilterType
+      [FieldMetadataType.TS_VECTOR, TSVectorFilterType],
     ]);
 
     return typeFilterMapping.get(fieldMetadataType);

packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts

@@ -158,6 +158,5 @@ export class CustomWorkspaceEntity extends BaseWorkspaceEntity {
   @WorkspaceIsNullable()
   @WorkspaceIsSystem()
   @WorkspaceFieldIndex({ indexType: IndexType.GIN })
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  searchVector: any;
+  searchVector: string;
 }

packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts

@@ -285,6 +285,5 @@ export class CompanyWorkspaceEntity extends BaseWorkspaceEntity {
   @WorkspaceIsNullable()
   @WorkspaceIsSystem()
   @WorkspaceFieldIndex({ indexType: IndexType.GIN })
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  searchVector: any;
+  searchVector: string;
 }

packages/twenty-server/src/modules/note/standard-objects/note.workspace-entity.ts

@@ -160,6 +160,5 @@ export class NoteWorkspaceEntity extends BaseWorkspaceEntity {
   @WorkspaceIsNullable()
   @WorkspaceIsSystem()
   @WorkspaceFieldIndex({ indexType: IndexType.GIN })
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  searchVector: any;
+  searchVector: string;
 }

packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts

@@ -240,6 +240,5 @@ export class OpportunityWorkspaceEntity extends BaseWorkspaceEntity {
   @WorkspaceIsNullable()
   @WorkspaceIsSystem()
   @WorkspaceFieldIndex({ indexType: IndexType.GIN })
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  searchVector: any;
+  searchVector: string;
 }

packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts

@@ -304,6 +304,5 @@ export class PersonWorkspaceEntity extends BaseWorkspaceEntity {
   @WorkspaceIsNullable()
   @WorkspaceIsSystem()
   @WorkspaceFieldIndex({ indexType: IndexType.GIN })
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  searchVector: any;
+  searchVector: string;
 }

packages/twenty-server/src/modules/task/standard-objects/task.workspace-entity.ts

@@ -215,6 +215,5 @@ export class TaskWorkspaceEntity extends BaseWorkspaceEntity {
   @WorkspaceIsNullable()
   @WorkspaceIsSystem()
   @WorkspaceFieldIndex({ indexType: IndexType.GIN })
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  searchVector: any;
+  searchVector: string;
 }

packages/twenty-server/src/modules/workspace-member/standard-objects/workspace-member.workspace-entity.ts

@@ -358,6 +358,5 @@ export class WorkspaceMemberWorkspaceEntity extends BaseWorkspaceEntity {
   @WorkspaceIsNullable()
   @WorkspaceIsSystem()
   @WorkspaceFieldIndex({ indexType: IndexType.GIN })
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  searchVector: any;
+  searchVector: string;
 }