diff --git a/docker-compose.yml b/docker-compose.yml index 39bac907..75240857 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3' - services: mysql: image: mysql:8.0 diff --git a/packages/cormo/lib/cjs/adapters/base.d.ts b/packages/cormo/lib/cjs/adapters/base.d.ts index 758666f5..d327f818 100644 --- a/packages/cormo/lib/cjs/adapters/base.d.ts +++ b/packages/cormo/lib/cjs/adapters/base.d.ts @@ -27,9 +27,11 @@ export interface Schemas { [table_name: string]: any; }; } +export type VectorOrderOption = Record | undefined>; export interface AdapterFindOptions { lean: boolean; orders: any[]; + vector_order?: VectorOrderOption; near?: any; select?: string[]; select_raw?: string[]; diff --git a/packages/cormo/lib/cjs/adapters/postgresql.js b/packages/cormo/lib/cjs/adapters/postgresql.js index 4ed8d70e..676678ca 100644 --- a/packages/cormo/lib/cjs/adapters/postgresql.js +++ b/packages/cormo/lib/cjs/adapters/postgresql.js @@ -65,6 +65,10 @@ function _typeToSQL(property) { return 'BIGINT'; case types.GeoPoint: return 'GEOMETRY(POINT)'; + case types.Vector: + return property.type.dimension + ? `VECTOR(${property.type.dimension})` + : 'VECTOR'; case types.Date: return 'TIMESTAMP WITHOUT TIME ZONE'; case types.Object: @@ -454,6 +458,7 @@ class PostgreSQLAdapter extends sql_base_js_1.SQLAdapterBase { callback(); }; this._pool.connect().then((client) => { + this._connection._logger.logQuery(sql, params); client .query(new QueryStream(sql, params)) .on('end', () => { @@ -648,10 +653,12 @@ class PostgreSQLAdapter extends sql_base_js_1.SQLAdapterBase { await this._connection._promise_connection; } if (transaction && transaction._adapter_connection) { + this._connection._logger.logQuery(text, values); transaction.checkFinished(); return await transaction._adapter_connection.query(text, values); } else { + this._connection._logger.logQuery(text, values); return await this._pool.query(text, values); } } @@ -670,6 +677,9 @@ class PostgreSQLAdapter extends sql_base_js_1.SQLAdapterBase { } return String(value); } + else if (property.type_class === types.Vector) { + return JSON.parse(value); + } return value; } /** @internal */ @@ -744,15 +754,17 @@ class PostgreSQLAdapter extends sql_base_js_1.SQLAdapterBase { column.udt_schema === 'public' && column.udt_name === 'geometry' ? new types.GeoPoint() - : column.data_type === 'timestamp without time zone' - ? new types.Date() - : column.data_type === 'json' - ? new types.Object() - : column.data_type === 'text' - ? new types.Text() - : column.data_type === 'bytea' - ? new types.Blob() - : undefined; + : column.data_type === 'vector' + ? new types.Vector() + : column.data_type === 'timestamp without time zone' + ? new types.Date() + : column.data_type === 'json' + ? new types.Object() + : column.data_type === 'text' + ? new types.Text() + : column.data_type === 'bytea' + ? new types.Blob() + : undefined; let adapter_type_string = column.data_type.toUpperCase(); if (column.data_type === 'character varying') { adapter_type_string += `(${column.character_maximum_length || 255})`; @@ -814,6 +826,16 @@ class PostgreSQLAdapter extends sql_base_js_1.SQLAdapterBase { fields.push(`"${dbname}"=ST_Point($${values.length - 1}, $${values.length})`); } } + else if (property.type_class === types.Vector) { + values.push(value != null ? JSON.stringify(value) : null); + if (insert) { + fields.push(`"${dbname}"`); + places.push(`$${values.length}`); + } + else { + fields.push(`"${dbname}"=$${values.length}`); + } + } else if (value && value.$inc != null) { values.push(value.$inc); fields.push(`"${dbname}"="${dbname}"+$${values.length}`); @@ -936,6 +958,39 @@ class PostgreSQLAdapter extends sql_base_js_1.SQLAdapterBase { } sql += ' ORDER BY ' + orders.join(','); } + else if (options.vector_order && + typeof options.vector_order === 'object' && + Object.keys(options.vector_order).length === 1) { + const column = Object.keys(options.vector_order)[0]; + const cond = options.vector_order[column]; + if (cond && typeof cond === 'object' && Object.keys(cond).length === 1) { + const key = Object.keys(cond)[0]; + const value = Object.values(cond)[0]; + let op = ''; + if (key === '$l2_distance') { + op = '<->'; + } + else if (key === '$l1_distance') { + op = '<+>'; + } + else if (key === '$cosine_distance') { + op = '<=>'; + } + else if (key === '$negative_inner_product') { + op = '<#>'; + } + else if (key === '$hamming_distance') { + op = '<~>'; + } + else if (key === '$jaccard_distance') { + op = '<%>'; + } + if (op) { + params.push(value != null ? JSON.stringify(value) : null); + sql += ` ORDER BY _Base."${column}" ${op} $${params.length}`; + } + } + } if (options.limit) { sql += ' LIMIT ' + options.limit; if (options.skip) { diff --git a/packages/cormo/lib/cjs/query.d.ts b/packages/cormo/lib/cjs/query.d.ts index eb56a631..e6671939 100644 --- a/packages/cormo/lib/cjs/query.d.ts +++ b/packages/cormo/lib/cjs/query.d.ts @@ -1,10 +1,12 @@ import stream from 'stream'; +import { VectorOrderOption } from './adapters/base.js'; import { BaseModel, ModelColumnNamesWithId } from './model/index.js'; import { Transaction } from './transaction.js'; import { RecordID } from './types.js'; interface QueryOptions { lean: boolean; orders?: string; + vector_order?: VectorOrderOption; near?: any; select_columns?: string[]; select_single: boolean; @@ -43,6 +45,7 @@ export interface QuerySingle extends PromiseLike select>(columns?: string): QuerySingle>; selectSingle>(column: K): QuerySingle; order(orders?: string): QuerySingle; + vector_order(order: VectorOrderOption): QuerySingle; group, F>(group_by: G | G[], fields?: F): QuerySingle>; @@ -95,6 +98,7 @@ interface QuerySingleNull extends PromiseLike>(columns?: string): QuerySingleNull>; selectSingle>(column: K): QuerySingleNull; order(orders?: string): QuerySingleNull; + vector_order(order: VectorOrderOption): QuerySingleNull; group, F>(group_by: G | G[], fields?: F): QuerySingleNull>; @@ -147,6 +151,7 @@ export interface QueryArray extends PromiseLike select>(columns?: string): QueryArray>; selectSingle>(column: K): QueryArray; order(orders?: string): QueryArray; + vector_order(order: VectorOrderOption): QueryArray; group, F>(group_by: G | G[], fields?: F): QueryArray>; @@ -238,6 +243,7 @@ declare class Query implements QuerySingle, Qu * Specifies orders of result */ order(orders?: string): this; + vector_order(order: VectorOrderOption): this; /** * Groups result records */ diff --git a/packages/cormo/lib/cjs/query.js b/packages/cormo/lib/cjs/query.js index 4a5a10ed..11671913 100644 --- a/packages/cormo/lib/cjs/query.js +++ b/packages/cormo/lib/cjs/query.js @@ -131,6 +131,13 @@ class Query { this._options.orders = orders; return this; } + vector_order(order) { + if (!this._current_if) { + return this; + } + this._options.vector_order = order; + return this; + } group(group_by, fields) { if (!this._current_if) { return this; @@ -595,6 +602,7 @@ class Query { node: this._options.node, index_hint: this._options.index_hint, orders, + vector_order: this._options.vector_order, skip: this._options.skip, transaction: this._options.transaction, distinct: this._options.distinct, diff --git a/packages/cormo/lib/cjs/types.d.ts b/packages/cormo/lib/cjs/types.d.ts index 5c229d85..bfe2331f 100644 --- a/packages/cormo/lib/cjs/types.d.ts +++ b/packages/cormo/lib/cjs/types.d.ts @@ -84,6 +84,22 @@ export interface CormoTypesGeoPointConstructor { (): CormoTypesGeoPoint; } declare const CormoTypesGeoPoint: CormoTypesGeoPointConstructor; +/** + * Represents a vector, used in model schemas. + * + * This type is supported only in PostgreSQL with pgvector + * @namespace types + * @class Vector + */ +export interface CormoTypesVector { + _type: 'vector'; + dimension?: number; +} +export interface CormoTypesVectorConstructor { + new (dimension?: number): CormoTypesVector; + (dimension?: number): CormoTypesVector; +} +declare const CormoTypesVector: CormoTypesVectorConstructor; /** * Represents a date, used in model schemas. * @namespace types @@ -152,17 +168,17 @@ export interface CormoTypesBlobConstructor { (): CormoTypesBlob; } declare const CormoTypesBlob: CormoTypesBlobConstructor; -export type ColumnTypeInternal = CormoTypesString | CormoTypesNumber | CormoTypesBoolean | CormoTypesDate | CormoTypesObject | CormoTypesInteger | CormoTypesBigInteger | CormoTypesGeoPoint | CormoTypesRecordID | CormoTypesText | CormoTypesBlob; -export type ColumnTypeInternalConstructor = CormoTypesStringConstructor | CormoTypesNumberConstructor | CormoTypesBooleanConstructor | CormoTypesDateConstructor | CormoTypesObjectConstructor | CormoTypesIntegerConstructor | CormoTypesBigIntegerConstructor | CormoTypesGeoPointConstructor | CormoTypesRecordIDConstructor | CormoTypesTextConstructor | CormoTypesBlobConstructor; +export type ColumnTypeInternal = CormoTypesString | CormoTypesNumber | CormoTypesBoolean | CormoTypesDate | CormoTypesObject | CormoTypesInteger | CormoTypesBigInteger | CormoTypesGeoPoint | CormoTypesVector | CormoTypesRecordID | CormoTypesText | CormoTypesBlob; +export type ColumnTypeInternalConstructor = CormoTypesStringConstructor | CormoTypesNumberConstructor | CormoTypesBooleanConstructor | CormoTypesDateConstructor | CormoTypesObjectConstructor | CormoTypesIntegerConstructor | CormoTypesBigIntegerConstructor | CormoTypesGeoPointConstructor | CormoTypesVectorConstructor | CormoTypesRecordIDConstructor | CormoTypesTextConstructor | CormoTypesBlobConstructor; type ColumnTypeNativeConstructor = StringConstructor | NumberConstructor | BooleanConstructor | DateConstructor | ObjectConstructor; -type ColumnTypeString = 'string' | 'number' | 'boolean' | 'date' | 'object' | 'integer' | 'biginteger' | 'geopoint' | 'recordid' | 'text' | 'blob'; +type ColumnTypeString = 'string' | 'number' | 'boolean' | 'date' | 'object' | 'integer' | 'biginteger' | 'geopoint' | 'vector' | 'recordid' | 'text' | 'blob'; export type ColumnType = ColumnTypeInternal | ColumnTypeInternalConstructor | ColumnTypeNativeConstructor | ColumnTypeString; /** * Converts JavaScript built-in class to CORMO type * @private */ declare function _toCORMOType(type: ColumnType): ColumnTypeInternal; -export { CormoTypesString as String, CormoTypesNumber as Number, CormoTypesBoolean as Boolean, CormoTypesInteger as Integer, CormoTypesBigInteger as BigInteger, CormoTypesGeoPoint as GeoPoint, CormoTypesDate as Date, CormoTypesObject as Object, CormoTypesRecordID as RecordID, CormoTypesText as Text, CormoTypesBlob as Blob, _toCORMOType, }; +export { CormoTypesString as String, CormoTypesNumber as Number, CormoTypesBoolean as Boolean, CormoTypesInteger as Integer, CormoTypesBigInteger as BigInteger, CormoTypesGeoPoint as GeoPoint, CormoTypesVector as Vector, CormoTypesDate as Date, CormoTypesObject as Object, CormoTypesRecordID as RecordID, CormoTypesText as Text, CormoTypesBlob as Blob, _toCORMOType, }; /** * A pseudo type represents a record's unique identifier. * diff --git a/packages/cormo/lib/cjs/types.js b/packages/cormo/lib/cjs/types.js index 9e0a0fdd..54b96f9f 100644 --- a/packages/cormo/lib/cjs/types.js +++ b/packages/cormo/lib/cjs/types.js @@ -5,7 +5,7 @@ * @namespace cormo */ Object.defineProperty(exports, "__esModule", { value: true }); -exports.Blob = exports.Text = exports.RecordID = exports.Object = exports.Date = exports.GeoPoint = exports.BigInteger = exports.Integer = exports.Boolean = exports.Number = exports.String = void 0; +exports.Blob = exports.Text = exports.RecordID = exports.Object = exports.Date = exports.Vector = exports.GeoPoint = exports.BigInteger = exports.Integer = exports.Boolean = exports.Number = exports.String = void 0; exports._toCORMOType = _toCORMOType; const CormoTypesString = function (length) { if (!(this instanceof CormoTypesString)) { @@ -50,6 +50,14 @@ const CormoTypesGeoPoint = function () { this.toString = () => 'geopoint'; }; exports.GeoPoint = CormoTypesGeoPoint; +const CormoTypesVector = function (dimension) { + if (!(this instanceof CormoTypesVector)) { + return new CormoTypesVector(dimension); + } + this.dimension = dimension; + this.toString = () => (this.dimension ? `vector(${this.dimension})` : 'vector'); +}; +exports.Vector = CormoTypesVector; const CormoTypesDate = function () { if (!(this instanceof CormoTypesDate)) { return new CormoTypesDate(); @@ -108,6 +116,8 @@ function _toCORMOType(type) { return new CormoTypesBigInteger(); case 'geopoint': return new CormoTypesGeoPoint(); + case 'vector': + return new CormoTypesVector(); case 'date': return new CormoTypesDate(); case 'object': diff --git a/packages/cormo/lib/esm/adapters/base.d.ts b/packages/cormo/lib/esm/adapters/base.d.ts index 758666f5..d327f818 100644 --- a/packages/cormo/lib/esm/adapters/base.d.ts +++ b/packages/cormo/lib/esm/adapters/base.d.ts @@ -27,9 +27,11 @@ export interface Schemas { [table_name: string]: any; }; } +export type VectorOrderOption = Record | undefined>; export interface AdapterFindOptions { lean: boolean; orders: any[]; + vector_order?: VectorOrderOption; near?: any; select?: string[]; select_raw?: string[]; diff --git a/packages/cormo/lib/esm/adapters/postgresql.js b/packages/cormo/lib/esm/adapters/postgresql.js index daecf07e..9d72c402 100644 --- a/packages/cormo/lib/esm/adapters/postgresql.js +++ b/packages/cormo/lib/esm/adapters/postgresql.js @@ -37,6 +37,10 @@ function _typeToSQL(property) { return 'BIGINT'; case types.GeoPoint: return 'GEOMETRY(POINT)'; + case types.Vector: + return property.type.dimension + ? `VECTOR(${property.type.dimension})` + : 'VECTOR'; case types.Date: return 'TIMESTAMP WITHOUT TIME ZONE'; case types.Object: @@ -426,6 +430,7 @@ export class PostgreSQLAdapter extends SQLAdapterBase { callback(); }; this._pool.connect().then((client) => { + this._connection._logger.logQuery(sql, params); client .query(new QueryStream(sql, params)) .on('end', () => { @@ -620,10 +625,12 @@ export class PostgreSQLAdapter extends SQLAdapterBase { await this._connection._promise_connection; } if (transaction && transaction._adapter_connection) { + this._connection._logger.logQuery(text, values); transaction.checkFinished(); return await transaction._adapter_connection.query(text, values); } else { + this._connection._logger.logQuery(text, values); return await this._pool.query(text, values); } } @@ -642,6 +649,9 @@ export class PostgreSQLAdapter extends SQLAdapterBase { } return String(value); } + else if (property.type_class === types.Vector) { + return JSON.parse(value); + } return value; } /** @internal */ @@ -716,15 +726,17 @@ export class PostgreSQLAdapter extends SQLAdapterBase { column.udt_schema === 'public' && column.udt_name === 'geometry' ? new types.GeoPoint() - : column.data_type === 'timestamp without time zone' - ? new types.Date() - : column.data_type === 'json' - ? new types.Object() - : column.data_type === 'text' - ? new types.Text() - : column.data_type === 'bytea' - ? new types.Blob() - : undefined; + : column.data_type === 'vector' + ? new types.Vector() + : column.data_type === 'timestamp without time zone' + ? new types.Date() + : column.data_type === 'json' + ? new types.Object() + : column.data_type === 'text' + ? new types.Text() + : column.data_type === 'bytea' + ? new types.Blob() + : undefined; let adapter_type_string = column.data_type.toUpperCase(); if (column.data_type === 'character varying') { adapter_type_string += `(${column.character_maximum_length || 255})`; @@ -786,6 +798,16 @@ export class PostgreSQLAdapter extends SQLAdapterBase { fields.push(`"${dbname}"=ST_Point($${values.length - 1}, $${values.length})`); } } + else if (property.type_class === types.Vector) { + values.push(value != null ? JSON.stringify(value) : null); + if (insert) { + fields.push(`"${dbname}"`); + places.push(`$${values.length}`); + } + else { + fields.push(`"${dbname}"=$${values.length}`); + } + } else if (value && value.$inc != null) { values.push(value.$inc); fields.push(`"${dbname}"="${dbname}"+$${values.length}`); @@ -908,6 +930,39 @@ export class PostgreSQLAdapter extends SQLAdapterBase { } sql += ' ORDER BY ' + orders.join(','); } + else if (options.vector_order && + typeof options.vector_order === 'object' && + Object.keys(options.vector_order).length === 1) { + const column = Object.keys(options.vector_order)[0]; + const cond = options.vector_order[column]; + if (cond && typeof cond === 'object' && Object.keys(cond).length === 1) { + const key = Object.keys(cond)[0]; + const value = Object.values(cond)[0]; + let op = ''; + if (key === '$l2_distance') { + op = '<->'; + } + else if (key === '$l1_distance') { + op = '<+>'; + } + else if (key === '$cosine_distance') { + op = '<=>'; + } + else if (key === '$negative_inner_product') { + op = '<#>'; + } + else if (key === '$hamming_distance') { + op = '<~>'; + } + else if (key === '$jaccard_distance') { + op = '<%>'; + } + if (op) { + params.push(value != null ? JSON.stringify(value) : null); + sql += ` ORDER BY _Base."${column}" ${op} $${params.length}`; + } + } + } if (options.limit) { sql += ' LIMIT ' + options.limit; if (options.skip) { diff --git a/packages/cormo/lib/esm/query.d.ts b/packages/cormo/lib/esm/query.d.ts index eb56a631..e6671939 100644 --- a/packages/cormo/lib/esm/query.d.ts +++ b/packages/cormo/lib/esm/query.d.ts @@ -1,10 +1,12 @@ import stream from 'stream'; +import { VectorOrderOption } from './adapters/base.js'; import { BaseModel, ModelColumnNamesWithId } from './model/index.js'; import { Transaction } from './transaction.js'; import { RecordID } from './types.js'; interface QueryOptions { lean: boolean; orders?: string; + vector_order?: VectorOrderOption; near?: any; select_columns?: string[]; select_single: boolean; @@ -43,6 +45,7 @@ export interface QuerySingle extends PromiseLike select>(columns?: string): QuerySingle>; selectSingle>(column: K): QuerySingle; order(orders?: string): QuerySingle; + vector_order(order: VectorOrderOption): QuerySingle; group, F>(group_by: G | G[], fields?: F): QuerySingle>; @@ -95,6 +98,7 @@ interface QuerySingleNull extends PromiseLike>(columns?: string): QuerySingleNull>; selectSingle>(column: K): QuerySingleNull; order(orders?: string): QuerySingleNull; + vector_order(order: VectorOrderOption): QuerySingleNull; group, F>(group_by: G | G[], fields?: F): QuerySingleNull>; @@ -147,6 +151,7 @@ export interface QueryArray extends PromiseLike select>(columns?: string): QueryArray>; selectSingle>(column: K): QueryArray; order(orders?: string): QueryArray; + vector_order(order: VectorOrderOption): QueryArray; group, F>(group_by: G | G[], fields?: F): QueryArray>; @@ -238,6 +243,7 @@ declare class Query implements QuerySingle, Qu * Specifies orders of result */ order(orders?: string): this; + vector_order(order: VectorOrderOption): this; /** * Groups result records */ diff --git a/packages/cormo/lib/esm/query.js b/packages/cormo/lib/esm/query.js index de727ad9..2f535dc0 100644 --- a/packages/cormo/lib/esm/query.js +++ b/packages/cormo/lib/esm/query.js @@ -125,6 +125,13 @@ class Query { this._options.orders = orders; return this; } + vector_order(order) { + if (!this._current_if) { + return this; + } + this._options.vector_order = order; + return this; + } group(group_by, fields) { if (!this._current_if) { return this; @@ -589,6 +596,7 @@ class Query { node: this._options.node, index_hint: this._options.index_hint, orders, + vector_order: this._options.vector_order, skip: this._options.skip, transaction: this._options.transaction, distinct: this._options.distinct, diff --git a/packages/cormo/lib/esm/types.d.ts b/packages/cormo/lib/esm/types.d.ts index 5c229d85..bfe2331f 100644 --- a/packages/cormo/lib/esm/types.d.ts +++ b/packages/cormo/lib/esm/types.d.ts @@ -84,6 +84,22 @@ export interface CormoTypesGeoPointConstructor { (): CormoTypesGeoPoint; } declare const CormoTypesGeoPoint: CormoTypesGeoPointConstructor; +/** + * Represents a vector, used in model schemas. + * + * This type is supported only in PostgreSQL with pgvector + * @namespace types + * @class Vector + */ +export interface CormoTypesVector { + _type: 'vector'; + dimension?: number; +} +export interface CormoTypesVectorConstructor { + new (dimension?: number): CormoTypesVector; + (dimension?: number): CormoTypesVector; +} +declare const CormoTypesVector: CormoTypesVectorConstructor; /** * Represents a date, used in model schemas. * @namespace types @@ -152,17 +168,17 @@ export interface CormoTypesBlobConstructor { (): CormoTypesBlob; } declare const CormoTypesBlob: CormoTypesBlobConstructor; -export type ColumnTypeInternal = CormoTypesString | CormoTypesNumber | CormoTypesBoolean | CormoTypesDate | CormoTypesObject | CormoTypesInteger | CormoTypesBigInteger | CormoTypesGeoPoint | CormoTypesRecordID | CormoTypesText | CormoTypesBlob; -export type ColumnTypeInternalConstructor = CormoTypesStringConstructor | CormoTypesNumberConstructor | CormoTypesBooleanConstructor | CormoTypesDateConstructor | CormoTypesObjectConstructor | CormoTypesIntegerConstructor | CormoTypesBigIntegerConstructor | CormoTypesGeoPointConstructor | CormoTypesRecordIDConstructor | CormoTypesTextConstructor | CormoTypesBlobConstructor; +export type ColumnTypeInternal = CormoTypesString | CormoTypesNumber | CormoTypesBoolean | CormoTypesDate | CormoTypesObject | CormoTypesInteger | CormoTypesBigInteger | CormoTypesGeoPoint | CormoTypesVector | CormoTypesRecordID | CormoTypesText | CormoTypesBlob; +export type ColumnTypeInternalConstructor = CormoTypesStringConstructor | CormoTypesNumberConstructor | CormoTypesBooleanConstructor | CormoTypesDateConstructor | CormoTypesObjectConstructor | CormoTypesIntegerConstructor | CormoTypesBigIntegerConstructor | CormoTypesGeoPointConstructor | CormoTypesVectorConstructor | CormoTypesRecordIDConstructor | CormoTypesTextConstructor | CormoTypesBlobConstructor; type ColumnTypeNativeConstructor = StringConstructor | NumberConstructor | BooleanConstructor | DateConstructor | ObjectConstructor; -type ColumnTypeString = 'string' | 'number' | 'boolean' | 'date' | 'object' | 'integer' | 'biginteger' | 'geopoint' | 'recordid' | 'text' | 'blob'; +type ColumnTypeString = 'string' | 'number' | 'boolean' | 'date' | 'object' | 'integer' | 'biginteger' | 'geopoint' | 'vector' | 'recordid' | 'text' | 'blob'; export type ColumnType = ColumnTypeInternal | ColumnTypeInternalConstructor | ColumnTypeNativeConstructor | ColumnTypeString; /** * Converts JavaScript built-in class to CORMO type * @private */ declare function _toCORMOType(type: ColumnType): ColumnTypeInternal; -export { CormoTypesString as String, CormoTypesNumber as Number, CormoTypesBoolean as Boolean, CormoTypesInteger as Integer, CormoTypesBigInteger as BigInteger, CormoTypesGeoPoint as GeoPoint, CormoTypesDate as Date, CormoTypesObject as Object, CormoTypesRecordID as RecordID, CormoTypesText as Text, CormoTypesBlob as Blob, _toCORMOType, }; +export { CormoTypesString as String, CormoTypesNumber as Number, CormoTypesBoolean as Boolean, CormoTypesInteger as Integer, CormoTypesBigInteger as BigInteger, CormoTypesGeoPoint as GeoPoint, CormoTypesVector as Vector, CormoTypesDate as Date, CormoTypesObject as Object, CormoTypesRecordID as RecordID, CormoTypesText as Text, CormoTypesBlob as Blob, _toCORMOType, }; /** * A pseudo type represents a record's unique identifier. * diff --git a/packages/cormo/lib/esm/types.js b/packages/cormo/lib/esm/types.js index fa2d0db6..554ed967 100644 --- a/packages/cormo/lib/esm/types.js +++ b/packages/cormo/lib/esm/types.js @@ -40,6 +40,13 @@ const CormoTypesGeoPoint = function () { } this.toString = () => 'geopoint'; }; +const CormoTypesVector = function (dimension) { + if (!(this instanceof CormoTypesVector)) { + return new CormoTypesVector(dimension); + } + this.dimension = dimension; + this.toString = () => (this.dimension ? `vector(${this.dimension})` : 'vector'); +}; const CormoTypesDate = function () { if (!(this instanceof CormoTypesDate)) { return new CormoTypesDate(); @@ -93,6 +100,8 @@ function _toCORMOType(type) { return new CormoTypesBigInteger(); case 'geopoint': return new CormoTypesGeoPoint(); + case 'vector': + return new CormoTypesVector(); case 'date': return new CormoTypesDate(); case 'object': @@ -126,4 +135,4 @@ function _toCORMOType(type) { } return type; } -export { CormoTypesString as String, CormoTypesNumber as Number, CormoTypesBoolean as Boolean, CormoTypesInteger as Integer, CormoTypesBigInteger as BigInteger, CormoTypesGeoPoint as GeoPoint, CormoTypesDate as Date, CormoTypesObject as Object, CormoTypesRecordID as RecordID, CormoTypesText as Text, CormoTypesBlob as Blob, _toCORMOType, }; +export { CormoTypesString as String, CormoTypesNumber as Number, CormoTypesBoolean as Boolean, CormoTypesInteger as Integer, CormoTypesBigInteger as BigInteger, CormoTypesGeoPoint as GeoPoint, CormoTypesVector as Vector, CormoTypesDate as Date, CormoTypesObject as Object, CormoTypesRecordID as RecordID, CormoTypesText as Text, CormoTypesBlob as Blob, _toCORMOType, }; diff --git a/packages/cormo/src/adapters/base.ts b/packages/cormo/src/adapters/base.ts index 1ae48f3b..9d5c15b1 100644 --- a/packages/cormo/src/adapters/base.ts +++ b/packages/cormo/src/adapters/base.ts @@ -30,9 +30,12 @@ export interface Schemas { foreign_keys?: { [table_name: string]: any }; } +export type VectorOrderOption = Record | undefined>; + export interface AdapterFindOptions { lean: boolean; orders: any[]; + vector_order?: VectorOrderOption; near?: any; select?: string[]; select_raw?: string[]; diff --git a/packages/cormo/src/adapters/postgresql.ts b/packages/cormo/src/adapters/postgresql.ts index ba6c400f..0eab457d 100644 --- a/packages/cormo/src/adapters/postgresql.ts +++ b/packages/cormo/src/adapters/postgresql.ts @@ -59,6 +59,10 @@ function _typeToSQL(property: ColumnPropertyInternal) { return 'BIGINT'; case types.GeoPoint: return 'GEOMETRY(POINT)'; + case types.Vector: + return (property.type as types.CormoTypesVector).dimension + ? `VECTOR(${(property.type as types.CormoTypesVector).dimension})` + : 'VECTOR'; case types.Date: return 'TIMESTAMP WITHOUT TIME ZONE'; case types.Object: @@ -490,6 +494,7 @@ export class PostgreSQLAdapter extends SQLAdapterBase { callback(); }; this._pool.connect().then((client: any) => { + this._connection._logger.logQuery(sql, params); client .query(new QueryStream(sql, params)) .on('end', () => { @@ -698,9 +703,11 @@ export class PostgreSQLAdapter extends SQLAdapterBase { await this._connection._promise_connection; } if (transaction && transaction._adapter_connection) { + this._connection._logger.logQuery(text, values); transaction.checkFinished(); return await transaction._adapter_connection.query(text, values); } else { + this._connection._logger.logQuery(text, values); return await this._pool.query(text, values); } } @@ -719,6 +726,8 @@ export class PostgreSQLAdapter extends SQLAdapterBase { return value.map((item: any) => (item ? String(item) : null)); } return String(value); + } else if (property.type_class === types.Vector) { + return JSON.parse(value); } return value; } @@ -797,15 +806,17 @@ export class PostgreSQLAdapter extends SQLAdapterBase { column.udt_schema === 'public' && column.udt_name === 'geometry' ? new types.GeoPoint() - : column.data_type === 'timestamp without time zone' - ? new types.Date() - : column.data_type === 'json' - ? new types.Object() - : column.data_type === 'text' - ? new types.Text() - : column.data_type === 'bytea' - ? new types.Blob() - : undefined; + : column.data_type === 'vector' + ? new types.Vector() + : column.data_type === 'timestamp without time zone' + ? new types.Date() + : column.data_type === 'json' + ? new types.Object() + : column.data_type === 'text' + ? new types.Text() + : column.data_type === 'bytea' + ? new types.Blob() + : undefined; let adapter_type_string = column.data_type.toUpperCase(); if (column.data_type === 'character varying') { adapter_type_string += `(${column.character_maximum_length || 255})`; @@ -875,6 +886,14 @@ export class PostgreSQLAdapter extends SQLAdapterBase { } else { fields.push(`"${dbname}"=ST_Point($${values.length - 1}, $${values.length})`); } + } else if (property.type_class === types.Vector) { + values.push(value != null ? JSON.stringify(value) : null); + if (insert) { + fields.push(`"${dbname}"`); + places.push(`$${values.length}`); + } else { + fields.push(`"${dbname}"=$${values.length}`); + } } else if (value && value.$inc != null) { values.push(value.$inc); fields.push(`"${dbname}"="${dbname}"+$${values.length}`); @@ -1005,6 +1024,35 @@ export class PostgreSQLAdapter extends SQLAdapterBase { orders.push(order_by); } sql += ' ORDER BY ' + orders.join(','); + } else if ( + options.vector_order && + typeof options.vector_order === 'object' && + Object.keys(options.vector_order).length === 1 + ) { + const column = Object.keys(options.vector_order)[0]; + const cond = options.vector_order[column]; + if (cond && typeof cond === 'object' && Object.keys(cond).length === 1) { + const key = Object.keys(cond)[0]; + const value = Object.values(cond)[0]; + let op = ''; + if (key === '$l2_distance') { + op = '<->'; + } else if (key === '$l1_distance') { + op = '<+>'; + } else if (key === '$cosine_distance') { + op = '<=>'; + } else if (key === '$negative_inner_product') { + op = '<#>'; + } else if (key === '$hamming_distance') { + op = '<~>'; + } else if (key === '$jaccard_distance') { + op = '<%>'; + } + if (op) { + params.push(value != null ? JSON.stringify(value) : null); + sql += ` ORDER BY _Base."${column}" ${op} $${params.length}`; + } + } } if (options.limit) { sql += ' LIMIT ' + options.limit; diff --git a/packages/cormo/src/query.ts b/packages/cormo/src/query.ts index cc8e5c63..b9296c57 100644 --- a/packages/cormo/src/query.ts +++ b/packages/cormo/src/query.ts @@ -1,7 +1,7 @@ import stream from 'stream'; import _ from 'lodash'; -import { AdapterBase, AdapterDeleteOptions, AdapterFindOptions } from './adapters/base.js'; +import { AdapterBase, AdapterDeleteOptions, AdapterFindOptions, VectorOrderOption } from './adapters/base.js'; import { Connection } from './connection/index.js'; import { BaseModel, ModelColumnNamesWithId } from './model/index.js'; import { Transaction } from './transaction.js'; @@ -10,6 +10,7 @@ import { RecordID } from './types.js'; interface QueryOptions { lean: boolean; orders?: string; + vector_order?: VectorOrderOption; near?: any; select_columns?: string[]; select_single: boolean; @@ -49,6 +50,7 @@ export interface QuerySingle extends PromiseLike select>(columns?: string): QuerySingle>; selectSingle>(column: K): QuerySingle; order(orders?: string): QuerySingle; + vector_order(order: VectorOrderOption): QuerySingle; group, F>( group_by: G | G[], fields?: F, @@ -96,6 +98,7 @@ interface QuerySingleNull extends PromiseLike>(columns?: string): QuerySingleNull>; selectSingle>(column: K): QuerySingleNull; order(orders?: string): QuerySingleNull; + vector_order(order: VectorOrderOption): QuerySingleNull; group, F>( group_by: G | G[], fields?: F, @@ -143,6 +146,7 @@ export interface QueryArray extends PromiseLike select>(columns?: string): QueryArray>; selectSingle>(column: K): QueryArray; order(orders?: string): QueryArray; + vector_order(order: VectorOrderOption): QueryArray; group, F>( group_by: G | G[], fields?: F, @@ -337,6 +341,14 @@ class Query implements QuerySingle, QueryArray return this; } + public vector_order(order: VectorOrderOption): this { + if (!this._current_if) { + return this; + } + this._options.vector_order = order; + return this; + } + /** * Groups result records */ @@ -846,6 +858,7 @@ class Query implements QuerySingle, QueryArray node: this._options.node, index_hint: this._options.index_hint, orders, + vector_order: this._options.vector_order, skip: this._options.skip, transaction: this._options.transaction, distinct: this._options.distinct, diff --git a/packages/cormo/src/types.ts b/packages/cormo/src/types.ts index 28eef034..342c89b0 100644 --- a/packages/cormo/src/types.ts +++ b/packages/cormo/src/types.ts @@ -134,6 +134,31 @@ const CormoTypesGeoPoint: CormoTypesGeoPointConstructor = function (this: CormoT this.toString = () => 'geopoint'; } as CormoTypesGeoPointConstructor; +/** + * Represents a vector, used in model schemas. + * + * This type is supported only in PostgreSQL with pgvector + * @namespace types + * @class Vector + */ +export interface CormoTypesVector { + _type: 'vector'; + dimension?: number; +} + +export interface CormoTypesVectorConstructor { + new (dimension?: number): CormoTypesVector; + (dimension?: number): CormoTypesVector; +} + +const CormoTypesVector: CormoTypesVectorConstructor = function (this: CormoTypesVector, dimension?: number): void { + if (!(this instanceof CormoTypesVector)) { + return new (CormoTypesVector as any)(dimension); + } + this.dimension = dimension; + this.toString = () => (this.dimension ? `vector(${this.dimension})` : 'vector'); +} as CormoTypesVectorConstructor; + /** * Represents a date, used in model schemas. * @namespace types @@ -251,6 +276,7 @@ export type ColumnTypeInternal = | CormoTypesInteger | CormoTypesBigInteger | CormoTypesGeoPoint + | CormoTypesVector | CormoTypesRecordID | CormoTypesText | CormoTypesBlob; @@ -264,6 +290,7 @@ export type ColumnTypeInternalConstructor = | CormoTypesIntegerConstructor | CormoTypesBigIntegerConstructor | CormoTypesGeoPointConstructor + | CormoTypesVectorConstructor | CormoTypesRecordIDConstructor | CormoTypesTextConstructor | CormoTypesBlobConstructor; @@ -284,6 +311,7 @@ type ColumnTypeString = | 'integer' | 'biginteger' | 'geopoint' + | 'vector' | 'recordid' | 'text' | 'blob'; @@ -317,6 +345,8 @@ function _toCORMOType(type: ColumnType): ColumnTypeInternal { return new CormoTypesBigInteger(); case 'geopoint': return new CormoTypesGeoPoint(); + case 'vector': + return new CormoTypesVector(); case 'date': return new CormoTypesDate(); case 'object': @@ -353,6 +383,7 @@ export { CormoTypesInteger as Integer, CormoTypesBigInteger as BigInteger, CormoTypesGeoPoint as GeoPoint, + CormoTypesVector as Vector, CormoTypesDate as Date, CormoTypesObject as Object, CormoTypesRecordID as RecordID, diff --git a/packages/cormo/test/cases/vector.ts b/packages/cormo/test/cases/vector.ts new file mode 100644 index 00000000..3920d985 --- /dev/null +++ b/packages/cormo/test/cases/vector.ts @@ -0,0 +1,47 @@ +import { expect } from 'chai'; +import * as cormo from '../../lib/esm/index.js'; + +export class DocumentRef extends cormo.BaseModel { + public name?: string; + public embedding?: number[]; +} + +export default function (models: { Document: typeof DocumentRef; connection: cormo.Connection | null }) { + it('create & find', async function () { + const record = await models.Document.create({ name: 'doc1', embedding: [1, 2, 3] }); + expect(record).to.have.deep.property('embedding', [1, 2, 3]); + const found = await models.Document.find(record.id); + expect(found).to.have.deep.property('embedding', [1, 2, 3]); + }); + + it('null value', async function () { + const record = await models.Document.create({ name: 'doc1', embedding: undefined }); + expect(record).to.have.deep.property('embedding', null); + const found = await models.Document.find(record.id); + expect(found).to.have.deep.property('embedding', null); + }); + + it('update', async function () { + const record = await models.Document.create({ name: 'doc1', embedding: [1, 2, 3] }); + await models.Document.find(record.id).update({ embedding: [4, 5, 6] }); + const found = await models.Document.find(record.id); + expect(found).to.have.deep.property('embedding', [4, 5, 6]); + }); + + it('l2 distance', async function () { + const records = await models.Document.createBulk([ + { name: 'doc1', embedding: [1, 2, 3] }, + { name: 'doc2', embedding: [4, 5, 6] }, + ]); + expect( + await models.Document.where() + .vector_order({ embedding: { $l2_distance: [3, 2, 4] } }) + .selectSingle('id'), + ).to.eql([records[0].id, records[1].id]); + expect( + await models.Document.where() + .vector_order({ embedding: { $l2_distance: [3, 4, 5] } }) + .selectSingle('id'), + ).to.eql([records[1].id, records[0].id]); + }); +} diff --git a/packages/cormo/test/vector.ts b/packages/cormo/test/vector.ts new file mode 100644 index 00000000..405b5a02 --- /dev/null +++ b/packages/cormo/test/vector.ts @@ -0,0 +1,48 @@ +import * as cormo from '../lib/esm/index.js'; +import cases, { DocumentRef } from './cases/vector.js'; +import _g from './support/common.js'; + +const _dbs = ['postgresql']; + +_dbs.forEach((db) => { + if (!_g.db_configs[db]) { + return; + } + + describe('vector-' + db, () => { + const models = { + Document: DocumentRef, + connection: null as cormo.Connection | null, + }; + + before(async () => { + _g.connection = models.connection = new cormo.Connection(db as any, _g.db_configs[db]); + + @cormo.Model() + // eslint-disable-next-line @typescript-eslint/no-shadow + class Document extends _g.BaseModel { + @cormo.Column('string') + public name?: string; + + @cormo.Column('vector') + public embedding?: number[]; + } + models.Document = Document; + + await _g.connection.dropAllModels(); + }); + + beforeEach(async () => { + await _g.deleteAllRecords([models.Document]); + }); + + after(async () => { + await models.connection!.dropAllModels(); + models.connection!.close(); + models.connection = null; + _g.connection = null; + }); + + cases(models); + }); +});