From 542832265b73a754099711b48c20aa1170934724 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 12 May 2025 12:23:30 +0200 Subject: [PATCH 01/26] Added pause / resume functionality to zongji listener --- index.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/index.js b/index.js index 02d7cbc..15b1a7e 100644 --- a/index.js +++ b/index.js @@ -284,6 +284,14 @@ ZongJi.prototype.stop = function () { }); }; +ZongJi.prototype.pause = function () { + this.connection.pause(); +}; + +ZongJi.prototype.resume = function () { + this.connection.resume(); +}; + // It includes every events by default. ZongJi.prototype._skipEvent = function (name) { const includes = this.filters.includeEvents; From f5b4ff6f3bb42caf79bcc1c88a942b5dfc0372b4 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 12 May 2025 12:46:25 +0200 Subject: [PATCH 02/26] Changeset --- .changeset/thick-carpets-shake.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/thick-carpets-shake.md diff --git a/.changeset/thick-carpets-shake.md b/.changeset/thick-carpets-shake.md new file mode 100644 index 0000000..c862614 --- /dev/null +++ b/.changeset/thick-carpets-shake.md @@ -0,0 +1,5 @@ +--- +'@powersync/mysql-zongji': minor +--- + +Added the functionality to pause / resume the zongji binlog listener From 32d7bb55038f654ea1aedad6093a2d997d62388e Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Tue, 13 May 2025 14:51:47 +0200 Subject: [PATCH 03/26] Added typescript types declarations Bumped node version to 22 --- .github/workflows/dev-packages.yaml | 2 +- .github/workflows/test.yml | 6 +- .nvmrc | 2 +- package.json | 1 + pnpm-lock.yaml | 15 +++- pnpm-workspace.yaml | 5 +- types/index.d.ts | 129 ++++++++++++++++++++++++++++ types/package.json | 13 +++ types/tsconfig.json | 13 +++ 9 files changed, 176 insertions(+), 10 deletions(-) create mode 100644 types/index.d.ts create mode 100644 types/package.json create mode 100644 types/tsconfig.json diff --git a/.github/workflows/dev-packages.yaml b/.github/workflows/dev-packages.yaml index 762c0b3..1be79df 100644 --- a/.github/workflows/dev-packages.yaml +++ b/.github/workflows/dev-packages.yaml @@ -17,7 +17,7 @@ jobs: uses: actions/setup-node@v4 with: node-version-file: '.nvmrc' - - uses: pnpm/action-setup@v2 + - uses: pnpm/action-setup@v4 name: Install pnpm with: version: 9 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 69d971a..ec1ba99 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,16 +26,14 @@ jobs: -e MYSQL_ROOT_PASSWORD=my_password \ -p 3306:3306 \ -d mysql:${{ matrix.mysql-version }} \ - --server-id=1 \ - --log-bin=/var/lib/mysql/mysql-bin.log \ - --binlog-format=row + --log-bin=/var/lib/mysql/mysql-bin.log - name: Setup NodeJS uses: actions/setup-node@v4 with: node-version-file: '.nvmrc' - - uses: pnpm/action-setup@v2 + - uses: pnpm/action-setup@v4 name: Install pnpm with: version: 9 diff --git a/.nvmrc b/.nvmrc index a81deba..517f386 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v20.12.2 +v22.14.0 diff --git a/package.json b/package.json index eacc420..09a625c 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ }, "scripts": { "release": "pnpm changeset publish", + "build": "cd types && tsc -b", "test": "tap run --disable-coverage --allow-incomplete-coverage -- --sequential test/suites.js", "format": "prettier --write ." }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6d0e5b2..9d37f24 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -28,6 +28,12 @@ importers: specifier: ^21.0.1 version: 21.0.1(@types/node@12.20.55)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4) + types: + devDependencies: + typescript: + specifier: ^5.8.3 + version: 5.8.3 + packages: '@alcalzone/ansi-tokenize@0.1.3': @@ -1497,6 +1503,11 @@ packages: engines: {node: '>=14.17'} hasBin: true + typescript@5.8.3: + resolution: {integrity: sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==} + engines: {node: '>=14.17'} + hasBin: true + unique-filename@3.0.0: resolution: {integrity: sha512-afXhuC55wkAmZ0P18QsVE6kp8JaxrEokN2HGIoIVv2ijHQd419H0+6EigAFcIzXeMIkcIkNBpB3L/DXB3cTS/g==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -3335,7 +3346,7 @@ snapshots: resolve-import: 2.0.0 rimraf: 6.0.1 sync-content: 2.0.1 - typescript: 5.5.4 + typescript: 5.8.3 walk-up-path: 4.0.0 tuf-js@2.2.1: @@ -3350,6 +3361,8 @@ snapshots: typescript@5.5.4: {} + typescript@5.8.3: {} + unique-filename@3.0.0: dependencies: unique-slug: 4.0.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 397c8cf..2b36c2b 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,3 @@ packages: - - './' - # exclude packages that are inside test directories - - '!test/*' \ No newline at end of file + - 'lib' + - 'types' \ No newline at end of file diff --git a/types/index.d.ts b/types/index.d.ts new file mode 100644 index 0000000..c19e9ee --- /dev/null +++ b/types/index.d.ts @@ -0,0 +1,129 @@ +import { Socket } from 'net'; + +export type ZongjiOptions = { + host: string; + user: string; + password: string; + dateStrings?: boolean; + timeZone?: string; +}; + +interface DatabaseFilter { + [databaseName: string]: string[] | true; +} + +export type StartOptions = { + includeEvents?: string[]; + excludeEvents?: string[]; + /** + * Describe which databases and tables to include (Only for row events). Use database names as the key and pass an array of table names or true (for the entire database). + * Example: { 'my_database': ['allow_table', 'another_table'], 'another_db': true } + */ + includeSchema?: DatabaseFilter; + /** + * Object describing which databases and tables to exclude (Same format as includeSchema) + * Example: { 'other_db': ['disallowed_table'], 'ex_db': true } + */ + excludeSchema?: DatabaseFilter; + /** + * BinLog position filename to start reading events from + */ + filename?: string; + /** + * BinLog position offset to start reading events from in file specified + */ + position?: number; + + /** + * Unique server ID for this replication client. + */ + serverId?: number; +}; + +export type ColumnSchema = { + COLUMN_NAME: string; + COLLATION_NAME: string; + CHARACTER_SET_NAME: string; + COLUMN_COMMENT: string; + COLUMN_TYPE: string; +}; + +export type ColumnDefinition = { + name: string; + charset: string; + type: number; + metadata: Record; +}; + +export type TableMapEntry = { + columnSchemas: ColumnSchema[]; + parentSchema: string; + tableName: string; + columns: ColumnDefinition[]; +}; + +export type BaseBinLogEvent = { + timestamp: number; + getEventName(): string; + + /** + * Next position in BinLog file to read from after + * this event. + */ + nextPosition: number; + /** + * Size of this event + */ + size: number; + flags: number; + useChecksum: boolean; +}; + +export type BinLogRotationEvent = BaseBinLogEvent & { + binlogName: string; + position: number; +}; + +export type BinLogGTIDLogEvent = BaseBinLogEvent & { + serverId: Buffer; + transactionRange: number; +}; + +export type BinLogXidEvent = BaseBinLogEvent & { + xid: number; +}; + +export type BinLogMutationEvent = BaseBinLogEvent & { + tableId: number; + numberOfColumns: number; + tableMap: Record; + rows: Record[]; +}; + +export type BinLogUpdateEvent = Omit & { + rows: { + before: Record; + after: Record; + }[]; +}; + +export type BinLogEvent = BinLogRotationEvent | BinLogGTIDLogEvent | BinLogXidEvent | BinLogMutationEvent; + +// @vlasky/mysql Connection +export interface MySQLConnection { + _socket?: Socket; + /** There are other forms of this method as well - this is the most basic one. */ + query(sql: string, callback: (error: any, results: any, fields: any) => void): void; +} + +export default class ZongJi { + connection: MySQLConnection; + constructor(options: ZongjiOptions); + + start(options: StartOptions): void; + stop(): void; + pause(): void; + resume(): void; + + on(type: 'binlog' | string, callback: (event: BinLogEvent) => void); +} diff --git a/types/package.json b/types/package.json new file mode 100644 index 0000000..5272790 --- /dev/null +++ b/types/package.json @@ -0,0 +1,13 @@ +{ + "name": "@powersync/types-mysql-zongji", + "version": "0.0.1", + "description": "Types package for Powersync MySQL ZongJi", + "main": "", + "types": "index.d.ts", + "scripts": { + "build": "tsc -b" + }, + "devDependencies": { + "typescript": "^5.8.3" + } +} diff --git a/types/tsconfig.json b/types/tsconfig.json new file mode 100644 index 0000000..ad00e3a --- /dev/null +++ b/types/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "declaration": true, + "emitDeclarationOnly": true, + "skipLibCheck": true, + "esModuleInterop": true + }, + "include": ["index.d.ts"] +} From 190cfa1f0a4ca1b8ee49307b5e31bcb081cab890 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Tue, 13 May 2025 14:59:40 +0200 Subject: [PATCH 04/26] Added changeset --- .changeset/thin-ties-pull.md | 5 +++++ types/package.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/thin-ties-pull.md diff --git a/.changeset/thin-ties-pull.md b/.changeset/thin-ties-pull.md new file mode 100644 index 0000000..5608a84 --- /dev/null +++ b/.changeset/thin-ties-pull.md @@ -0,0 +1,5 @@ +--- +'@powersync/types-mysql-zongji': minor +--- + +First release of Zongji types diff --git a/types/package.json b/types/package.json index 5272790..6e186b0 100644 --- a/types/package.json +++ b/types/package.json @@ -1,6 +1,6 @@ { "name": "@powersync/types-mysql-zongji", - "version": "0.0.1", + "version": "0.0.0", "description": "Types package for Powersync MySQL ZongJi", "main": "", "types": "index.d.ts", From c0dd69604c41e496ca7fc83761fb959a10be0a86 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 14 May 2025 09:35:23 +0200 Subject: [PATCH 05/26] Fixed workspace package structure --- pnpm-workspace.yaml | 6 ++++-- types/package.json | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2b36c2b..4cd8f24 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,3 +1,5 @@ packages: - - 'lib' - - 'types' \ No newline at end of file + - './' + - 'types' + # exclude packages that are inside test directories + - '!test/*' \ No newline at end of file diff --git a/types/package.json b/types/package.json index 6e186b0..5272790 100644 --- a/types/package.json +++ b/types/package.json @@ -1,6 +1,6 @@ { "name": "@powersync/types-mysql-zongji", - "version": "0.0.0", + "version": "0.0.1", "description": "Types package for Powersync MySQL ZongJi", "main": "", "types": "index.d.ts", From fae16f9499b8a9beb6abbed5ee46980e661d986a Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 14 May 2025 09:44:27 +0200 Subject: [PATCH 06/26] Made types package public --- types/package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/types/package.json b/types/package.json index 5272790..38e4f3e 100644 --- a/types/package.json +++ b/types/package.json @@ -4,6 +4,9 @@ "description": "Types package for Powersync MySQL ZongJi", "main": "", "types": "index.d.ts", + "publishConfig": { + "access": "public" + }, "scripts": { "build": "tsc -b" }, From 3a850f7fab96fc8292035c0047a54ed7b6b349ee Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 14 May 2025 10:03:06 +0200 Subject: [PATCH 07/26] No default export for types package --- types/index.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/index.d.ts b/types/index.d.ts index c19e9ee..a6986b2 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -116,7 +116,7 @@ export interface MySQLConnection { query(sql: string, callback: (error: any, results: any, fields: any) => void): void; } -export default class ZongJi { +export class ZongJi { connection: MySQLConnection; constructor(options: ZongjiOptions); From ed630649292b28da478031473ba77da34f780b91 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 14 May 2025 10:40:35 +0200 Subject: [PATCH 08/26] Integrated types with main package --- .changeset/thick-carpets-shake.md | 1 + .changeset/thin-ties-pull.md | 5 ----- package.json | 2 +- pnpm-lock.yaml | 6 ------ pnpm-workspace.yaml | 1 - types/package.json | 16 ---------------- types/tsconfig.json | 13 ------------- 7 files changed, 2 insertions(+), 42 deletions(-) delete mode 100644 .changeset/thin-ties-pull.md delete mode 100644 types/package.json delete mode 100644 types/tsconfig.json diff --git a/.changeset/thick-carpets-shake.md b/.changeset/thick-carpets-shake.md index c862614..afef7bd 100644 --- a/.changeset/thick-carpets-shake.md +++ b/.changeset/thick-carpets-shake.md @@ -3,3 +3,4 @@ --- Added the functionality to pause / resume the zongji binlog listener +Added type definitions \ No newline at end of file diff --git a/.changeset/thin-ties-pull.md b/.changeset/thin-ties-pull.md deleted file mode 100644 index 5608a84..0000000 --- a/.changeset/thin-ties-pull.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@powersync/types-mysql-zongji': minor ---- - -First release of Zongji types diff --git a/package.json b/package.json index 09a625c..c7f63a5 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "0.1.0", "description": "A MySQL 8.0 >= compatible fork of ZongJi - a MySQL binlog listener for Node.js.", "main": "index.js", + "types": "types/index.d.ts", "directories": { "test": "test" }, @@ -11,7 +12,6 @@ }, "scripts": { "release": "pnpm changeset publish", - "build": "cd types && tsc -b", "test": "tap run --disable-coverage --allow-incomplete-coverage -- --sequential test/suites.js", "format": "prettier --write ." }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9d37f24..faa8992 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -28,12 +28,6 @@ importers: specifier: ^21.0.1 version: 21.0.1(@types/node@12.20.55)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4) - types: - devDependencies: - typescript: - specifier: ^5.8.3 - version: 5.8.3 - packages: '@alcalzone/ansi-tokenize@0.1.3': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 4cd8f24..397c8cf 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,5 +1,4 @@ packages: - './' - - 'types' # exclude packages that are inside test directories - '!test/*' \ No newline at end of file diff --git a/types/package.json b/types/package.json deleted file mode 100644 index 38e4f3e..0000000 --- a/types/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "@powersync/types-mysql-zongji", - "version": "0.0.1", - "description": "Types package for Powersync MySQL ZongJi", - "main": "", - "types": "index.d.ts", - "publishConfig": { - "access": "public" - }, - "scripts": { - "build": "tsc -b" - }, - "devDependencies": { - "typescript": "^5.8.3" - } -} diff --git a/types/tsconfig.json b/types/tsconfig.json deleted file mode 100644 index ad00e3a..0000000 --- a/types/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "strict": true, - "declaration": true, - "emitDeclarationOnly": true, - "skipLibCheck": true, - "esModuleInterop": true - }, - "include": ["index.d.ts"] -} From ad88a6086e189665a29992f984f2b11c26ee9f7f Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 14 May 2025 12:08:14 +0200 Subject: [PATCH 09/26] Made project into a module --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index c7f63a5..0f4a3e3 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "description": "A MySQL 8.0 >= compatible fork of ZongJi - a MySQL binlog listener for Node.js.", "main": "index.js", "types": "types/index.d.ts", + "type": "module", "directories": { "test": "test" }, From 9b0adfe232453b6a65fbb741e2a735f088222e63 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 14 May 2025 13:30:29 +0200 Subject: [PATCH 10/26] Revert module flag --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 0f4a3e3..c7f63a5 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,6 @@ "description": "A MySQL 8.0 >= compatible fork of ZongJi - a MySQL binlog listener for Node.js.", "main": "index.js", "types": "types/index.d.ts", - "type": "module", "directories": { "test": "test" }, From d7ffe63614bfa0a9c2f84898cf2f5f3a745f4075 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Fri, 16 May 2025 09:47:44 +0200 Subject: [PATCH 11/26] Try declare class for Zongji class type --- types/index.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/index.d.ts b/types/index.d.ts index a6986b2..ae9fe08 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -116,7 +116,7 @@ export interface MySQLConnection { query(sql: string, callback: (error: any, results: any, fields: any) => void): void; } -export class ZongJi { +export declare class ZongJi { connection: MySQLConnection; constructor(options: ZongjiOptions); From e6189930bfc24c52ce3927e527fa57d76540aa56 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Fri, 16 May 2025 10:02:00 +0200 Subject: [PATCH 12/26] Try with export default --- types/index.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/index.d.ts b/types/index.d.ts index ae9fe08..c19e9ee 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -116,7 +116,7 @@ export interface MySQLConnection { query(sql: string, callback: (error: any, results: any, fields: any) => void): void; } -export declare class ZongJi { +export default class ZongJi { connection: MySQLConnection; constructor(options: ZongjiOptions); From b52fdeeedad893a969259cf48fdf3c8ec0300267 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 19 May 2025 09:37:52 +0200 Subject: [PATCH 13/26] Try with named export --- index.js | 4 +++- types/index.d.ts | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 15b1a7e..dd06450 100644 --- a/index.js +++ b/index.js @@ -319,4 +319,6 @@ ZongJi.prototype._skipSchema = function (database, table) { return excluded || !included; }; -module.exports = ZongJi; +module.exports = { + ZongJi +}; diff --git a/types/index.d.ts b/types/index.d.ts index c19e9ee..feca2b3 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -116,7 +116,7 @@ export interface MySQLConnection { query(sql: string, callback: (error: any, results: any, fields: any) => void): void; } -export default class ZongJi { +export declare class ZongJi { connection: MySQLConnection; constructor(options: ZongjiOptions); @@ -125,5 +125,5 @@ export default class ZongJi { pause(): void; resume(): void; - on(type: 'binlog' | string, callback: (event: BinLogEvent) => void); + on(type: 'binlog' | string, callback: (event: BinLogEvent) => void): void; } From aa4b533ac6455495bd6e357322d09ce6ed42fcc5 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 19 May 2025 09:52:05 +0200 Subject: [PATCH 14/26] Try both named and default export --- index.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index dd06450..340a8c7 100644 --- a/index.js +++ b/index.js @@ -319,6 +319,5 @@ ZongJi.prototype._skipSchema = function (database, table) { return excluded || !included; }; -module.exports = { - ZongJi -}; +module.exports = ZongJi; +module.exports.ZongJi = ZongJi; From 53329b888c61d84602502aecd3b33bfffe858494 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Tue, 20 May 2025 13:05:01 +0200 Subject: [PATCH 15/26] Added binlog tablemap event type definition --- types/index.d.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/types/index.d.ts b/types/index.d.ts index feca2b3..9caf91e 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -75,6 +75,10 @@ export type BaseBinLogEvent = { * Size of this event */ size: number; + /** + * Unique identifier for table that this event relates to. This ID changes when a table is altered. + */ + tableId: number; flags: number; useChecksum: boolean; }; @@ -107,7 +111,22 @@ export type BinLogUpdateEvent = Omit & { }[]; }; -export type BinLogEvent = BinLogRotationEvent | BinLogGTIDLogEvent | BinLogXidEvent | BinLogMutationEvent; +export type BinLogTableMapEvent = BaseBinLogEvent & { + schemaName: string; + tableName: string; + columnCount: number; + /** + * Column types for this table. This is a list of MySQL column type ids + */ + columnTypes: number[]; +}; + +export type BinLogEvent = + | BinLogRotationEvent + | BinLogGTIDLogEvent + | BinLogXidEvent + | BinLogMutationEvent + | BinLogTableMapEvent; // @vlasky/mysql Connection export interface MySQLConnection { From e1a7d1c4904dd1aae7b483c397032a889fa1f109 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Tue, 20 May 2025 15:10:15 +0200 Subject: [PATCH 16/26] Only emit table map events on table changes. --- index.js | 3 +-- types/index.d.ts | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 340a8c7..dc7cd2a 100644 --- a/index.js +++ b/index.js @@ -240,9 +240,8 @@ ZongJi.prototype.start = function (options = {}) { this.emit('binlog', event); this.connection.resume(); }); - return; } - break; + return; } case 'Rotate': if (this.options.filename !== event.binlogName) { diff --git a/types/index.d.ts b/types/index.d.ts index 9caf91e..89e59c3 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -76,7 +76,8 @@ export type BaseBinLogEvent = { */ size: number; /** - * Unique identifier for table that this event relates to. This ID changes when a table is altered. + * Identifier for table that this event relates to. This value can change between server restarts and can be reused, so it is not a reliable identifier. + * Also, changes when a table schema has been altered. */ tableId: number; flags: number; From 6feffcd75b9fd4ecdf969494d9fe5c9df93cc71e Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 21 May 2025 11:24:48 +0200 Subject: [PATCH 17/26] Slight adjustments to types to correctly reflect binlog event structure. Updated types tests to ignore table map events. --- test/events.js | 2 +- test/types.js | 12 ++++++------ types/index.d.ts | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/events.js b/test/events.js index 4991afd..7841057 100644 --- a/test/events.js +++ b/test/events.js @@ -37,7 +37,7 @@ tap.test('Binlog option startAtEnd', (test) => { test.test(`prepare new table ${TEST_TABLE}`, (test) => { testDb.execute( [ - 'FLUSH LOGS', // Ensure ZongJi perserveres through a rotation event + 'FLUSH LOGS', // Ensure ZongJi perseveres through a rotation event `DROP TABLE IF EXISTS ${TEST_TABLE}`, `CREATE TABLE ${TEST_TABLE} (col INT UNSIGNED)`, `INSERT INTO ${TEST_TABLE} (col) VALUES (12)` diff --git a/test/types.js b/test/types.js index 27e6b8b..8f18323 100644 --- a/test/types.js +++ b/test/types.js @@ -44,7 +44,12 @@ function defineTypeTest(name, fields, testRows, customTest) { includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows'], serverId: testDb.serverId() }); - zongji.on('binlog', (event) => eventLog.push(event)); + zongji.on('binlog', (event) => { + // Ignore TableMap events for types test + if (event.getTypeName() !== 'TableMap') { + eventLog.push(event); + } + }); zongji.on('error', (error) => errorLog.push(error)); zongji.on('ready', () => { testDb.execute(testQueries, (error, results) => { @@ -67,11 +72,6 @@ function defineTypeTest(name, fields, testRows, customTest) { test, eventLog, [ - { - _type: 'TableMap', - tableName: TEST_TABLE, - schemaName: testDb.SCHEMA_NAME - }, expectedWrite ], testRows.length, diff --git a/types/index.d.ts b/types/index.d.ts index 89e59c3..3b9f1a6 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -99,7 +99,6 @@ export type BinLogXidEvent = BaseBinLogEvent & { }; export type BinLogMutationEvent = BaseBinLogEvent & { - tableId: number; numberOfColumns: number; tableMap: Record; rows: Record[]; @@ -113,6 +112,7 @@ export type BinLogUpdateEvent = Omit & { }; export type BinLogTableMapEvent = BaseBinLogEvent & { + tableMap: Record; schemaName: string; tableName: string; columnCount: number; From ec6bfa40f82be827a380ee3bda718b48e258ae47 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 22 May 2025 13:09:09 +0200 Subject: [PATCH 18/26] Cleaned up defined types, and added descriptions Bumped dependencies --- README.md | 7 +- docker-test.sh | 2 +- package.json | 8 +- pnpm-lock.yaml | 284 ++++++++++++++++++++--------------------------- types/index.d.ts | 47 ++++++-- 5 files changed, 166 insertions(+), 182 deletions(-) diff --git a/README.md b/README.md index a1af4a2..ca6929a 100644 --- a/README.md +++ b/README.md @@ -143,8 +143,9 @@ Neither method requires any arguments. ## Run Tests -- install [Docker](https://www.docker.com/community-edition#download) -- run `docker-compose up` and then `./docker-test.sh` +- Install [Docker](https://www.docker.com/community-edition#download) +- Run `docker-compose up` and then `./docker-test.sh` +- Some tests can be run directly, but some of the types.js tests pertaining to dates are affected by the host's timezone and are consequently brittle. ## References @@ -157,7 +158,7 @@ The following resources provided valuable information that greatly assisted in c - https://kkaefer.com/node-cpp-modules/ - https://dev.mysql.com/doc/internals/en/replication-protocol.html - https://web.archive.org/web/20200201195450/https://www.cs.wichita.edu/~chang/lecture/cs742/program/how-mysql-c-api.html -- https://github.com/jeremycole/mysql_binlog (Ruby implemenation of MySQL binlog parser) +- https://github.com/jeremycole/mysql_binlog (Ruby implementation of MySQL binlog parser) - https://dev.mysql.com/doc/internals/en/date-and-time-data-type-representation.html ## License diff --git a/docker-test.sh b/docker-test.sh index 46d5445..485ae40 100755 --- a/docker-test.sh +++ b/docker-test.sh @@ -2,6 +2,6 @@ MYSQL_HOSTS="mysql80 mysql84" for hostname in ${MYSQL_HOSTS}; do - echo $hostname + node 20 + echo $hostname + node 22 docker run -it --network=powersync-mysql-zongji_default -e MYSQL_HOST=$hostname -w /build -v $PWD:/build node:20 npm test done diff --git a/package.json b/package.json index c7f63a5..72a5f31 100644 --- a/package.json +++ b/package.json @@ -36,15 +36,15 @@ ], "license": "MIT", "engines": { - "node": ">=20.0.0" + "node": ">=22.0.0" }, "devDependencies": { - "@changesets/cli": "^2.27.3", - "prettier": "^2.8.8", + "@changesets/cli": "^2.29.4", + "prettier": "^3.5.3", "tap": "^21.0.1" }, "dependencies": { - "big-integer": "1.6.51", + "big-integer": "1.6.52", "iconv-lite": "0.6.3", "@vlasky/mysql": "^2.18.6" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index faa8992..1207dff 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,18 +12,18 @@ importers: specifier: ^2.18.6 version: 2.18.6 big-integer: - specifier: 1.6.51 - version: 1.6.51 + specifier: 1.6.52 + version: 1.6.52 iconv-lite: specifier: 0.6.3 version: 0.6.3 devDependencies: '@changesets/cli': - specifier: ^2.27.3 - version: 2.27.8 + specifier: ^2.29.4 + version: 2.29.4 prettier: - specifier: ^2.8.8 - version: 2.8.8 + specifier: ^3.5.3 + version: 3.5.3 tap: specifier: ^21.0.1 version: 21.0.1(@types/node@12.20.55)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4) @@ -43,60 +43,60 @@ packages: '@bcoe/v8-coverage@0.2.3': resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} - '@changesets/apply-release-plan@7.0.5': - resolution: {integrity: sha512-1cWCk+ZshEkSVEZrm2fSj1Gz8sYvxgUL4Q78+1ZZqeqfuevPTPk033/yUZ3df8BKMohkqqHfzj0HOOrG0KtXTw==} + '@changesets/apply-release-plan@7.0.12': + resolution: {integrity: sha512-EaET7As5CeuhTzvXTQCRZeBUcisoYPDDcXvgTE/2jmmypKp0RC7LxKj/yzqeh/1qFTZI7oDGFcL1PHRuQuketQ==} - '@changesets/assemble-release-plan@6.0.4': - resolution: {integrity: sha512-nqICnvmrwWj4w2x0fOhVj2QEGdlUuwVAwESrUo5HLzWMI1rE5SWfsr9ln+rDqWB6RQ2ZyaMZHUcU7/IRaUJS+Q==} + '@changesets/assemble-release-plan@6.0.8': + resolution: {integrity: sha512-y8+8LvZCkKJdbUlpXFuqcavpzJR80PN0OIfn8HZdwK7Sh6MgLXm4hKY5vu6/NDoKp8lAlM4ERZCqRMLxP4m+MQ==} - '@changesets/changelog-git@0.2.0': - resolution: {integrity: sha512-bHOx97iFI4OClIT35Lok3sJAwM31VbUM++gnMBV16fdbtBhgYu4dxsphBF/0AZZsyAHMrnM0yFcj5gZM1py6uQ==} + '@changesets/changelog-git@0.2.1': + resolution: {integrity: sha512-x/xEleCFLH28c3bQeQIyeZf8lFXyDFVn1SgcBiR2Tw/r4IAWlk1fzxCEZ6NxQAjF2Nwtczoen3OA2qR+UawQ8Q==} - '@changesets/cli@2.27.8': - resolution: {integrity: sha512-gZNyh+LdSsI82wBSHLQ3QN5J30P4uHKJ4fXgoGwQxfXwYFTJzDdvIJasZn8rYQtmKhyQuiBj4SSnLuKlxKWq4w==} + '@changesets/cli@2.29.4': + resolution: {integrity: sha512-VW30x9oiFp/un/80+5jLeWgEU6Btj8IqOgI+X/zAYu4usVOWXjPIK5jSSlt5jsCU7/6Z7AxEkarxBxGUqkAmNg==} hasBin: true - '@changesets/config@3.0.3': - resolution: {integrity: sha512-vqgQZMyIcuIpw9nqFIpTSNyc/wgm/Lu1zKN5vECy74u95Qx/Wa9g27HdgO4NkVAaq+BGA8wUc/qvbvVNs93n6A==} + '@changesets/config@3.1.1': + resolution: {integrity: sha512-bd+3Ap2TKXxljCggI0mKPfzCQKeV/TU4yO2h2C6vAihIo8tzseAn2e7klSuiyYYXvgu53zMN1OeYMIQkaQoWnA==} '@changesets/errors@0.2.0': resolution: {integrity: sha512-6BLOQUscTpZeGljvyQXlWOItQyU71kCdGz7Pi8H8zdw6BI0g3m43iL4xKUVPWtG+qrrL9DTjpdn8eYuCQSRpow==} - '@changesets/get-dependents-graph@2.1.2': - resolution: {integrity: sha512-sgcHRkiBY9i4zWYBwlVyAjEM9sAzs4wYVwJUdnbDLnVG3QwAaia1Mk5P8M7kraTOZN+vBET7n8KyB0YXCbFRLQ==} + '@changesets/get-dependents-graph@2.1.3': + resolution: {integrity: sha512-gphr+v0mv2I3Oxt19VdWRRUxq3sseyUpX9DaHpTUmLj92Y10AGy+XOtV+kbM6L/fDcpx7/ISDFK6T8A/P3lOdQ==} - '@changesets/get-release-plan@4.0.4': - resolution: {integrity: sha512-SicG/S67JmPTrdcc9Vpu0wSQt7IiuN0dc8iR5VScnnTVPfIaLvKmEGRvIaF0kcn8u5ZqLbormZNTO77bCEvyWw==} + '@changesets/get-release-plan@4.0.12': + resolution: {integrity: sha512-KukdEgaafnyGryUwpHG2kZ7xJquOmWWWk5mmoeQaSvZTWH1DC5D/Sw6ClgGFYtQnOMSQhgoEbDxAbpIIayKH1g==} '@changesets/get-version-range-type@0.4.0': resolution: {integrity: sha512-hwawtob9DryoGTpixy1D3ZXbGgJu1Rhr+ySH2PvTLHvkZuQ7sRT4oQwMh0hbqZH1weAooedEjRsbrWcGLCeyVQ==} - '@changesets/git@3.0.1': - resolution: {integrity: sha512-pdgHcYBLCPcLd82aRcuO0kxCDbw/yISlOtkmwmE8Odo1L6hSiZrBOsRl84eYG7DRCab/iHnOkWqExqc4wxk2LQ==} + '@changesets/git@3.0.4': + resolution: {integrity: sha512-BXANzRFkX+XcC1q/d27NKvlJ1yf7PSAgi8JG6dt8EfbHFHi4neau7mufcSca5zRhwOL8j9s6EqsxmT+s+/E6Sw==} '@changesets/logger@0.1.1': resolution: {integrity: sha512-OQtR36ZlnuTxKqoW4Sv6x5YIhOmClRd5pWsjZsddYxpWs517R0HkyiefQPIytCVh4ZcC5x9XaG8KTdd5iRQUfg==} - '@changesets/parse@0.4.0': - resolution: {integrity: sha512-TS/9KG2CdGXS27S+QxbZXgr8uPsP4yNJYb4BC2/NeFUj80Rni3TeD2qwWmabymxmrLo7JEsytXH1FbpKTbvivw==} + '@changesets/parse@0.4.1': + resolution: {integrity: sha512-iwksMs5Bf/wUItfcg+OXrEpravm5rEd9Bf4oyIPL4kVTmJQ7PNDSd6MDYkpSJR1pn7tz/k8Zf2DhTCqX08Ou+Q==} - '@changesets/pre@2.0.1': - resolution: {integrity: sha512-vvBJ/If4jKM4tPz9JdY2kGOgWmCowUYOi5Ycv8dyLnEE8FgpYYUo1mgJZxcdtGGP3aG8rAQulGLyyXGSLkIMTQ==} + '@changesets/pre@2.0.2': + resolution: {integrity: sha512-HaL/gEyFVvkf9KFg6484wR9s0qjAXlZ8qWPDkTyKF6+zqjBe/I2mygg3MbpZ++hdi0ToqNUF8cjj7fBy0dg8Ug==} - '@changesets/read@0.6.1': - resolution: {integrity: sha512-jYMbyXQk3nwP25nRzQQGa1nKLY0KfoOV7VLgwucI0bUO8t8ZLCr6LZmgjXsiKuRDc+5A6doKPr9w2d+FEJ55zQ==} + '@changesets/read@0.6.5': + resolution: {integrity: sha512-UPzNGhsSjHD3Veb0xO/MwvasGe8eMyNrR/sT9gR8Q3DhOQZirgKhhXv/8hVsI0QpPjR004Z9iFxoJU6in3uGMg==} - '@changesets/should-skip-package@0.1.1': - resolution: {integrity: sha512-H9LjLbF6mMHLtJIc/eHR9Na+MifJ3VxtgP/Y+XLn4BF7tDTEN1HNYtH6QMcjP1uxp9sjaFYmW8xqloaCi/ckTg==} + '@changesets/should-skip-package@0.1.2': + resolution: {integrity: sha512-qAK/WrqWLNCP22UDdBTMPH5f41elVDlsNyat180A33dWxuUDyNpg6fPi/FyTZwRriVjg0L8gnjJn2F9XAoF0qw==} '@changesets/types@4.1.0': resolution: {integrity: sha512-LDQvVDv5Kb50ny2s25Fhm3d9QSZimsoUGBsUioj6MC3qbMUCuC8GPIvk/M6IvXx3lYhAs0lwWUQLb+VIEUCECw==} - '@changesets/types@6.0.0': - resolution: {integrity: sha512-b1UkfNulgKoWfqyHtzKS5fOZYSJO+77adgL7DLRDr+/7jhChN+QcHnbjiQVOz/U+Ts3PGNySq7diAItzDgugfQ==} + '@changesets/types@6.1.0': + resolution: {integrity: sha512-rKQcJ+o1nKNgeoYRHKOS07tAMNd3YSN0uHaJOZYjBAgxfV7TUE7JE+z4BzZdQwb5hKaYbayKN5KrYV7ODb2rAA==} - '@changesets/write@0.3.2': - resolution: {integrity: sha512-kDxDrPNpUgsjDbWBvUo27PzKX4gqeKOlhibaOXDJA6kuBisGqNHv/HwGJrAu8U/dSf8ZEFIeHIPtvSlZI1kULw==} + '@changesets/write@0.4.0': + resolution: {integrity: sha512-CdTLvIOPiCNuH71pyDu3rA+Q0n65cmAbXnwWH84rKGiFumFzkmHNT8KHTMEchcxN+Kl8I54xGUhJ7l3E7X396Q==} '@cspotcode/source-map-support@0.8.1': resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} @@ -385,9 +385,6 @@ packages: '@types/node@12.20.55': resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} - '@types/semver@7.5.8': - resolution: {integrity: sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==} - '@vlasky/mysql@2.18.6': resolution: {integrity: sha512-c+qz/zzqecteQLchoje0E0rjLla935d6hHPpMKmfyQJnHlycLpR49ekS6s/zUAt8w0Um5hFglKXm4+PeJTVhaQ==} engines: {node: '>= 0.6'} @@ -466,8 +463,8 @@ packages: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} engines: {node: '>=4'} - big-integer@1.6.51: - resolution: {integrity: sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==} + big-integer@1.6.52: + resolution: {integrity: sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==} engines: {node: '>=0.6'} bignumber.js@9.1.1: @@ -558,13 +555,14 @@ packages: core-util-is@1.0.2: resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} - cross-spawn@5.1.0: - resolution: {integrity: sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==} - cross-spawn@7.0.3: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} engines: {node: '>= 8'} + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + debug@4.1.1: resolution: {integrity: sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==} deprecated: Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797) @@ -734,9 +732,6 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} - graceful-fs@4.2.3: - resolution: {integrity: sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==} - has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} @@ -759,8 +754,9 @@ packages: resolution: {integrity: sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==} engines: {node: '>= 14'} - human-id@1.0.2: - resolution: {integrity: sha512-UNopramDEhHJD+VR+ehk8rOslwSfByxPIZyJRfV739NDhN5LF1fa1MqnzKm2lGTQRjNrjK19Q5fhkgIfjlVUKw==} + human-id@4.1.1: + resolution: {integrity: sha512-3gKm/gCSUipeLsRYZbbdA1BD83lBoWUkZ7G9VFrhWPAU76KwYo5KR8V28bpoPm/ygy0x5/GCbpRQdY7VLYCoIg==} + hasBin: true iconv-lite@0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} @@ -940,9 +936,6 @@ packages: resolution: {integrity: sha512-CgeuL5uom6j/ZVrg7G/+1IXqRY8JXX4Hghfy5YE0EhoYQWvndP1kufu58cmZLNIDKnRhZrXfdS9urVWx98AipQ==} engines: {node: 20 || >=22} - lru-cache@4.1.5: - resolution: {integrity: sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==} - make-dir@4.0.0: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} @@ -1186,6 +1179,11 @@ packages: engines: {node: '>=10.13.0'} hasBin: true + prettier@3.5.3: + resolution: {integrity: sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==} + engines: {node: '>=14'} + hasBin: true + prismjs-terminal@1.2.3: resolution: {integrity: sha512-xc0zuJ5FMqvW+DpiRkvxURlz98DdfDsZcFHdO699+oL+ykbFfgI7O4VDEgUyc07BSL2NHl3zdb8m/tZ/aaqUrw==} engines: {node: '>=16'} @@ -1217,9 +1215,6 @@ packages: resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==} engines: {node: '>=10'} - pseudomap@1.0.2: - resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} - queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -1310,25 +1305,14 @@ packages: engines: {node: '>=10'} hasBin: true - shebang-command@1.2.0: - resolution: {integrity: sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==} - engines: {node: '>=0.10.0'} - shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} - shebang-regex@1.0.0: - resolution: {integrity: sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==} - engines: {node: '>=0.10.0'} - shebang-regex@3.0.0: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} - signal-exit@3.0.2: - resolution: {integrity: sha512-meQNNykwecVxdu1RlYMKpQx4+wefIYpmxi6gexo/KAbwquJrBUrBmKYJrE8KFkVQAAVWEnwNdu21PgrD77J3xA==} - signal-exit@3.0.7: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} @@ -1364,8 +1348,8 @@ packages: resolution: {integrity: sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==} engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} - spawndamnit@2.0.0: - resolution: {integrity: sha512-j4JKEcncSjFlqIwU5L/rp2N5SIPsdxaRsIv678+TZxZ0SRDJTm8JrxJMjE/XuiEZNEir3S8l0Fa3Ke339WI4qA==} + spawndamnit@3.0.1: + resolution: {integrity: sha512-MmnduQUuHCoFckZoWnXsTg7JaiLBJrKFj9UI2MbRPGaJeVpsLcVBu6P/IGZovziM/YBsellCmsprgNA+w0CzVg==} spdx-correct@3.1.0: resolution: {integrity: sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==} @@ -1539,10 +1523,6 @@ packages: resolution: {integrity: sha512-3hu+tD8YzSLGuFYtPRb48vdhKMi0KQV5sn+uWr8+7dMEq/2G/dtLrdDinkLjqq5TIbIBjYJ4Ax/n3YiaW7QM8A==} engines: {node: 20 || >=22} - which@1.3.1: - resolution: {integrity: sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==} - hasBin: true - which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -1585,9 +1565,6 @@ packages: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} - yallist@2.1.2: - resolution: {integrity: sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==} - yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} @@ -1632,13 +1609,13 @@ snapshots: '@bcoe/v8-coverage@0.2.3': {} - '@changesets/apply-release-plan@7.0.5': + '@changesets/apply-release-plan@7.0.12': dependencies: - '@changesets/config': 3.0.3 + '@changesets/config': 3.1.1 '@changesets/get-version-range-type': 0.4.0 - '@changesets/git': 3.0.1 - '@changesets/should-skip-package': 0.1.1 - '@changesets/types': 6.0.0 + '@changesets/git': 3.0.4 + '@changesets/should-skip-package': 0.1.2 + '@changesets/types': 6.1.0 '@manypkg/get-packages': 1.1.3 detect-indent: 6.1.0 fs-extra: 7.0.1 @@ -1648,58 +1625,56 @@ snapshots: resolve-from: 5.0.0 semver: 7.6.3 - '@changesets/assemble-release-plan@6.0.4': + '@changesets/assemble-release-plan@6.0.8': dependencies: '@changesets/errors': 0.2.0 - '@changesets/get-dependents-graph': 2.1.2 - '@changesets/should-skip-package': 0.1.1 - '@changesets/types': 6.0.0 + '@changesets/get-dependents-graph': 2.1.3 + '@changesets/should-skip-package': 0.1.2 + '@changesets/types': 6.1.0 '@manypkg/get-packages': 1.1.3 semver: 7.6.3 - '@changesets/changelog-git@0.2.0': + '@changesets/changelog-git@0.2.1': dependencies: - '@changesets/types': 6.0.0 + '@changesets/types': 6.1.0 - '@changesets/cli@2.27.8': + '@changesets/cli@2.29.4': dependencies: - '@changesets/apply-release-plan': 7.0.5 - '@changesets/assemble-release-plan': 6.0.4 - '@changesets/changelog-git': 0.2.0 - '@changesets/config': 3.0.3 + '@changesets/apply-release-plan': 7.0.12 + '@changesets/assemble-release-plan': 6.0.8 + '@changesets/changelog-git': 0.2.1 + '@changesets/config': 3.1.1 '@changesets/errors': 0.2.0 - '@changesets/get-dependents-graph': 2.1.2 - '@changesets/get-release-plan': 4.0.4 - '@changesets/git': 3.0.1 + '@changesets/get-dependents-graph': 2.1.3 + '@changesets/get-release-plan': 4.0.12 + '@changesets/git': 3.0.4 '@changesets/logger': 0.1.1 - '@changesets/pre': 2.0.1 - '@changesets/read': 0.6.1 - '@changesets/should-skip-package': 0.1.1 - '@changesets/types': 6.0.0 - '@changesets/write': 0.3.2 + '@changesets/pre': 2.0.2 + '@changesets/read': 0.6.5 + '@changesets/should-skip-package': 0.1.2 + '@changesets/types': 6.1.0 + '@changesets/write': 0.4.0 '@manypkg/get-packages': 1.1.3 - '@types/semver': 7.5.8 ansi-colors: 4.1.3 ci-info: 3.9.0 enquirer: 2.4.1 external-editor: 3.1.0 fs-extra: 7.0.1 mri: 1.2.0 - outdent: 0.5.0 p-limit: 2.2.2 package-manager-detector: 0.2.0 picocolors: 1.1.0 resolve-from: 5.0.0 semver: 7.6.3 - spawndamnit: 2.0.0 + spawndamnit: 3.0.1 term-size: 2.2.1 - '@changesets/config@3.0.3': + '@changesets/config@3.1.1': dependencies: '@changesets/errors': 0.2.0 - '@changesets/get-dependents-graph': 2.1.2 + '@changesets/get-dependents-graph': 2.1.3 '@changesets/logger': 0.1.1 - '@changesets/types': 6.0.0 + '@changesets/types': 6.1.0 '@manypkg/get-packages': 1.1.3 fs-extra: 7.0.1 micromatch: 4.0.8 @@ -1708,72 +1683,72 @@ snapshots: dependencies: extendable-error: 0.1.7 - '@changesets/get-dependents-graph@2.1.2': + '@changesets/get-dependents-graph@2.1.3': dependencies: - '@changesets/types': 6.0.0 + '@changesets/types': 6.1.0 '@manypkg/get-packages': 1.1.3 picocolors: 1.1.0 semver: 7.6.3 - '@changesets/get-release-plan@4.0.4': + '@changesets/get-release-plan@4.0.12': dependencies: - '@changesets/assemble-release-plan': 6.0.4 - '@changesets/config': 3.0.3 - '@changesets/pre': 2.0.1 - '@changesets/read': 0.6.1 - '@changesets/types': 6.0.0 + '@changesets/assemble-release-plan': 6.0.8 + '@changesets/config': 3.1.1 + '@changesets/pre': 2.0.2 + '@changesets/read': 0.6.5 + '@changesets/types': 6.1.0 '@manypkg/get-packages': 1.1.3 '@changesets/get-version-range-type@0.4.0': {} - '@changesets/git@3.0.1': + '@changesets/git@3.0.4': dependencies: '@changesets/errors': 0.2.0 '@manypkg/get-packages': 1.1.3 is-subdir: 1.2.0 micromatch: 4.0.8 - spawndamnit: 2.0.0 + spawndamnit: 3.0.1 '@changesets/logger@0.1.1': dependencies: picocolors: 1.1.0 - '@changesets/parse@0.4.0': + '@changesets/parse@0.4.1': dependencies: - '@changesets/types': 6.0.0 + '@changesets/types': 6.1.0 js-yaml: 3.13.1 - '@changesets/pre@2.0.1': + '@changesets/pre@2.0.2': dependencies: '@changesets/errors': 0.2.0 - '@changesets/types': 6.0.0 + '@changesets/types': 6.1.0 '@manypkg/get-packages': 1.1.3 fs-extra: 7.0.1 - '@changesets/read@0.6.1': + '@changesets/read@0.6.5': dependencies: - '@changesets/git': 3.0.1 + '@changesets/git': 3.0.4 '@changesets/logger': 0.1.1 - '@changesets/parse': 0.4.0 - '@changesets/types': 6.0.0 + '@changesets/parse': 0.4.1 + '@changesets/types': 6.1.0 fs-extra: 7.0.1 p-filter: 2.1.0 picocolors: 1.1.0 - '@changesets/should-skip-package@0.1.1': + '@changesets/should-skip-package@0.1.2': dependencies: - '@changesets/types': 6.0.0 + '@changesets/types': 6.1.0 '@manypkg/get-packages': 1.1.3 '@changesets/types@4.1.0': {} - '@changesets/types@6.0.0': {} + '@changesets/types@6.1.0': {} - '@changesets/write@0.3.2': + '@changesets/write@0.4.0': dependencies: - '@changesets/types': 6.0.0 + '@changesets/types': 6.1.0 fs-extra: 7.0.1 - human-id: 1.0.2 + human-id: 4.1.1 prettier: 2.8.8 '@cspotcode/source-map-support@0.8.1': @@ -2216,8 +2191,6 @@ snapshots: '@types/node@12.20.55': {} - '@types/semver@7.5.8': {} - '@vlasky/mysql@2.18.6': dependencies: bignumber.js: 9.1.1 @@ -2283,7 +2256,7 @@ snapshots: dependencies: is-windows: 1.0.2 - big-integer@1.6.51: {} + big-integer@1.6.52: {} bignumber.js@9.1.1: {} @@ -2381,13 +2354,13 @@ snapshots: core-util-is@1.0.2: {} - cross-spawn@5.1.0: + cross-spawn@7.0.3: dependencies: - lru-cache: 4.1.5 - shebang-command: 1.2.0 - which: 1.3.1 + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 - cross-spawn@7.0.3: + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 shebang-command: 2.0.0 @@ -2488,13 +2461,13 @@ snapshots: fs-extra@7.0.1: dependencies: - graceful-fs: 4.2.3 + graceful-fs: 4.2.11 jsonfile: 4.0.0 universalify: 0.1.2 fs-extra@8.1.0: dependencies: - graceful-fs: 4.2.3 + graceful-fs: 4.2.11 jsonfile: 4.0.0 universalify: 0.1.2 @@ -2548,8 +2521,6 @@ snapshots: graceful-fs@4.2.11: {} - graceful-fs@4.2.3: {} - has-flag@4.0.0: {} hosted-git-info@7.0.2: @@ -2574,7 +2545,7 @@ snapshots: transitivePeerDependencies: - supports-color - human-id@1.0.2: {} + human-id@4.1.1: {} iconv-lite@0.4.24: dependencies: @@ -2714,7 +2685,7 @@ snapshots: jsonfile@4.0.0: optionalDependencies: - graceful-fs: 4.2.3 + graceful-fs: 4.2.11 jsonparse@1.3.1: {} @@ -2738,11 +2709,6 @@ snapshots: lru-cache@11.0.1: {} - lru-cache@4.1.5: - dependencies: - pseudomap: 1.0.2 - yallist: 2.1.2 - make-dir@4.0.0: dependencies: semver: 7.6.3 @@ -2995,6 +2961,8 @@ snapshots: prettier@2.8.8: {} + prettier@3.5.3: {} + prismjs-terminal@1.2.3: dependencies: chalk: 5.3.0 @@ -3018,8 +2986,6 @@ snapshots: err-code: 2.0.3 retry: 0.12.0 - pseudomap@1.0.2: {} - queue-microtask@1.2.3: {} react-dom@18.3.1(react@18.3.1): @@ -3050,7 +3016,7 @@ snapshots: read-yaml-file@1.1.0: dependencies: - graceful-fs: 4.2.3 + graceful-fs: 4.2.11 js-yaml: 3.13.1 pify: 4.0.1 strip-bom: 3.0.0 @@ -3110,20 +3076,12 @@ snapshots: semver@7.6.3: {} - shebang-command@1.2.0: - dependencies: - shebang-regex: 1.0.0 - shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 - shebang-regex@1.0.0: {} - shebang-regex@3.0.0: {} - signal-exit@3.0.2: {} - signal-exit@3.0.7: {} signal-exit@4.1.0: {} @@ -3166,10 +3124,10 @@ snapshots: ip-address: 9.0.5 smart-buffer: 4.2.0 - spawndamnit@2.0.0: + spawndamnit@3.0.1: dependencies: - cross-spawn: 5.1.0 - signal-exit: 3.0.2 + cross-spawn: 7.0.6 + signal-exit: 4.1.0 spdx-correct@3.1.0: dependencies: @@ -3388,10 +3346,6 @@ snapshots: walk-up-path@4.0.0: {} - which@1.3.1: - dependencies: - isexe: 2.0.0 - which@2.0.2: dependencies: isexe: 2.0.0 @@ -3426,8 +3380,6 @@ snapshots: y18n@5.0.8: {} - yallist@2.1.2: {} - yallist@4.0.0: {} yaml-types@0.4.0(yaml@2.5.1): diff --git a/types/index.d.ts b/types/index.d.ts index 3b9f1a6..b00528c 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -1,5 +1,14 @@ import { Socket } from 'net'; +/** + * The types described here were added on an adhoc basis based on requirements in the Powersync Service. + * They are not complete. + */ + +/** + * Connection options for the MySQL connection that Zongji will use. For a complete list and description see: + * https://www.npmjs.com/package/@vlasky/mysql#connection-options + */ export type ZongjiOptions = { host: string; user: string; @@ -8,11 +17,18 @@ export type ZongjiOptions = { timeZone?: string; }; +/** + * Record specifying a database and specific tables. ie. ['MyDatabase']: ['table1', 'table2'] + * Alternatively specifying true will include all tables in the database. + */ interface DatabaseFilter { [databaseName: string]: string[] | true; } export type StartOptions = { + /** + * List specifying which binlog events to listen for. When not specified, all events are included. + */ includeEvents?: string[]; excludeEvents?: string[]; /** @@ -51,6 +67,10 @@ export type ColumnSchema = { export type ColumnDefinition = { name: string; charset: string; + /** + * MySQl column type constant. For a list of type mappings see: + * https://github.com/mysqljs/mysql/blob/master/lib/protocol/constants/types.js + */ type: number; metadata: Record; }; @@ -67,12 +87,11 @@ export type BaseBinLogEvent = { getEventName(): string; /** - * Next position in BinLog file to read from after - * this event. + * Next position in BinLog file to read from after this event. */ nextPosition: number; /** - * Size of this event + * Size in bytes of this event */ size: number; /** @@ -98,13 +117,20 @@ export type BinLogXidEvent = BaseBinLogEvent & { xid: number; }; -export type BinLogMutationEvent = BaseBinLogEvent & { +export type BinLogRowEvent = BaseBinLogEvent & { + /** + * The number of columns affected by this row event + */ numberOfColumns: number; - tableMap: Record; + /** + * TableMap describing the current schema. Format: + * [TableId]: TableMapEntry + */ + tableMap: Record; rows: Record[]; }; -export type BinLogUpdateEvent = Omit & { +export type BinLogRowUpdateEvent = Omit & { rows: { before: Record; after: Record; @@ -112,7 +138,11 @@ export type BinLogUpdateEvent = Omit & { }; export type BinLogTableMapEvent = BaseBinLogEvent & { - tableMap: Record; + /** + * TableMap describing the current schema. Format: + * [TableId]: TableMapEntry + */ + tableMap: Record; schemaName: string; tableName: string; columnCount: number; @@ -126,7 +156,8 @@ export type BinLogEvent = | BinLogRotationEvent | BinLogGTIDLogEvent | BinLogXidEvent - | BinLogMutationEvent + | BinLogRowEvent + | BinLogRowUpdateEvent | BinLogTableMapEvent; // @vlasky/mysql Connection From b755c41bfd7cfaa61efe942ba5eb35df3d7a511b Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 22 May 2025 16:41:22 +0200 Subject: [PATCH 19/26] Added potential race condition safety net to not start the binlog streaming if the listener has already been stopped --- index.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index dc7cd2a..0e02392 100644 --- a/index.js +++ b/index.js @@ -22,6 +22,7 @@ function ZongJi(dsn) { this.ctrlCallbacks = []; this.tableMap = {}; this.ready = false; + this.stopped = false; this.useChecksum = false; this._establishConnection(dsn); @@ -261,11 +262,13 @@ ZongJi.prototype.start = function (options = {}) { Promise.all(promises) .then(() => { - this.BinlogClass = initBinlogClass(this); - this.ready = true; - this.emit('ready'); + if (!this.stopped) { + this.BinlogClass = initBinlogClass(this); + this.ready = true; + this.emit('ready'); - this.connection._protocol._enqueue(new this.BinlogClass(binlogHandler)); + this.connection._protocol._enqueue(new this.BinlogClass(binlogHandler)); + } }) .catch((err) => { this.emit('error', err); @@ -281,6 +284,7 @@ ZongJi.prototype.stop = function () { } this.emit('stopped'); }); + this.stopped = true; }; ZongJi.prototype.pause = function () { From 2e6288f0015890937d81b95eece228450eca93d9 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 26 May 2025 12:52:49 +0200 Subject: [PATCH 20/26] Added another stop check before starting the listener --- index.js | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/index.js b/index.js index 0e02392..3cac381 100644 --- a/index.js +++ b/index.js @@ -192,6 +192,10 @@ ZongJi.prototype.start = function (options = {}) { this._options(options); this._filters(options); + // Don't start listener if it has already explicitly been stopped + if (this.stopped) { + return; + } const testChecksum = (resolve, reject) => { this._isChecksumEnabled((err, checksumEnabled) => { if (err) { @@ -254,25 +258,26 @@ ZongJi.prototype.start = function (options = {}) { this.emit('binlog', event); }; - let promises = [new Promise(testChecksum)]; + if (!this.stopped) { + let promises = [new Promise(testChecksum)]; - if (this.options.startAtEnd) { - promises.push(new Promise(findBinlogEnd)); - } + if (this.options.startAtEnd) { + promises.push(new Promise(findBinlogEnd)); + } - Promise.all(promises) - .then(() => { - if (!this.stopped) { + Promise.all(promises) + .then(() => { this.BinlogClass = initBinlogClass(this); this.ready = true; this.emit('ready'); - - this.connection._protocol._enqueue(new this.BinlogClass(binlogHandler)); - } - }) - .catch((err) => { - this.emit('error', err); - }); + if (!this.stopped) { + this.connection._protocol._enqueue(new this.BinlogClass(binlogHandler)); + } + }) + .catch((err) => { + this.emit('error', err); + }); + } }; ZongJi.prototype.stop = function () { From 3c569425e91ab770529bd09df51dcb3770c44f95 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 26 May 2025 14:11:36 +0200 Subject: [PATCH 21/26] Don't emit errors on an already stopped listener. --- index.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 3cac381..898ba11 100644 --- a/index.js +++ b/index.js @@ -268,14 +268,17 @@ ZongJi.prototype.start = function (options = {}) { Promise.all(promises) .then(() => { this.BinlogClass = initBinlogClass(this); - this.ready = true; - this.emit('ready'); if (!this.stopped) { this.connection._protocol._enqueue(new this.BinlogClass(binlogHandler)); + this.ready = true; + this.emit('ready'); } }) .catch((err) => { - this.emit('error', err); + // Don't emit errors if the listener is already stopped + if (!this.stopped) { + this.emit('error', err); + } }); } }; From 76462dde98d5a968f0028163d837181b2abb1157 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 26 May 2025 16:36:56 +0200 Subject: [PATCH 22/26] Exposed port configuration in types. Removed superfluous stop checks. --- index.js | 61 +++++++++++++++++++++--------------------------- types/index.d.ts | 2 ++ 2 files changed, 29 insertions(+), 34 deletions(-) diff --git a/index.js b/index.js index 898ba11..7b46713 100644 --- a/index.js +++ b/index.js @@ -192,10 +192,6 @@ ZongJi.prototype.start = function (options = {}) { this._options(options); this._filters(options); - // Don't start listener if it has already explicitly been stopped - if (this.stopped) { - return; - } const testChecksum = (resolve, reject) => { this._isChecksumEnabled((err, checksumEnabled) => { if (err) { @@ -258,41 +254,38 @@ ZongJi.prototype.start = function (options = {}) { this.emit('binlog', event); }; - if (!this.stopped) { - let promises = [new Promise(testChecksum)]; + let promises = [new Promise(testChecksum)]; - if (this.options.startAtEnd) { - promises.push(new Promise(findBinlogEnd)); - } - - Promise.all(promises) - .then(() => { - this.BinlogClass = initBinlogClass(this); - if (!this.stopped) { - this.connection._protocol._enqueue(new this.BinlogClass(binlogHandler)); - this.ready = true; - this.emit('ready'); - } - }) - .catch((err) => { - // Don't emit errors if the listener is already stopped - if (!this.stopped) { - this.emit('error', err); - } - }); + if (this.options.startAtEnd) { + promises.push(new Promise(findBinlogEnd)); } + + Promise.all(promises) + .then(() => { + this.BinlogClass = initBinlogClass(this); + if (!this.stopped) { + this.connection._protocol._enqueue(new this.BinlogClass(binlogHandler)); + this.ready = true; + this.emit('ready'); + } + }) + .catch((err) => { + this.emit('error', err); + }); }; ZongJi.prototype.stop = function () { - // Binary log connection does not end with destroy() - this.connection.destroy(); - this.ctrlConnection.query('KILL ' + this.connection.threadId, () => { - if (this.ctrlConnectionOwner) { - this.ctrlConnection.destroy(); - } - this.emit('stopped'); - }); - this.stopped = true; + if (!this.stopped) { + this.stopped = true; + // Binary log connection does not end with destroy() + this.connection.destroy(); + this.ctrlConnection.query('KILL ' + this.connection.threadId, () => { + if (this.ctrlConnectionOwner) { + this.ctrlConnection.destroy(); + } + this.emit('stopped'); + }); + } }; ZongJi.prototype.pause = function () { diff --git a/types/index.d.ts b/types/index.d.ts index b00528c..eb0e4c9 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -11,6 +11,7 @@ import { Socket } from 'net'; */ export type ZongjiOptions = { host: string; + port?: number; user: string; password: string; dateStrings?: boolean; @@ -168,6 +169,7 @@ export interface MySQLConnection { } export declare class ZongJi { + stopped: boolean; connection: MySQLConnection; constructor(options: ZongjiOptions); From 36a65a71054972178b7df5b8b45e48dd8d094c5a Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Tue, 27 May 2025 10:51:02 +0200 Subject: [PATCH 23/26] Don't run startup checks if listener was already stopped --- index.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 7b46713..abeb6e3 100644 --- a/index.js +++ b/index.js @@ -193,6 +193,9 @@ ZongJi.prototype.start = function (options = {}) { this._filters(options); const testChecksum = (resolve, reject) => { + if (this.stopped) { + resolve(); + } this._isChecksumEnabled((err, checksumEnabled) => { if (err) { reject(err); @@ -204,6 +207,9 @@ ZongJi.prototype.start = function (options = {}) { }; const findBinlogEnd = (resolve, reject) => { + if (this.stopped) { + resolve(); + } this._findBinlogEnd((err, result) => { if (err) { return reject(err); @@ -279,7 +285,7 @@ ZongJi.prototype.stop = function () { this.stopped = true; // Binary log connection does not end with destroy() this.connection.destroy(); - this.ctrlConnection.query('KILL ' + this.connection.threadId, () => { + this.ctrlConnection.query('KILL ' + this.connection.threadId, (err, result) => { if (this.ctrlConnectionOwner) { this.ctrlConnection.destroy(); } From d99e149bd52dd5a33923348261191005fc613523 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Tue, 27 May 2025 14:51:26 +0200 Subject: [PATCH 24/26] Updated changeset --- .changeset/thick-carpets-shake.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.changeset/thick-carpets-shake.md b/.changeset/thick-carpets-shake.md index afef7bd..d849e4e 100644 --- a/.changeset/thick-carpets-shake.md +++ b/.changeset/thick-carpets-shake.md @@ -3,4 +3,6 @@ --- Added the functionality to pause / resume the zongji binlog listener -Added type definitions \ No newline at end of file +Improved stop/start functionality to better handle some race conditions +Added type definitions +The first time a TableMap event is emitted, it will always be emitted, but subsequent events will only be emitted if the table map has changed. \ No newline at end of file From c95b15799a6675a44c1024eecb3fe45a0be0b4ae Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 28 May 2025 12:39:19 +0200 Subject: [PATCH 25/26] Added test Added stopped check safeguards for pause and resume functions --- index.js | 8 ++++-- test/events.js | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index abeb6e3..318fb97 100644 --- a/index.js +++ b/index.js @@ -295,11 +295,15 @@ ZongJi.prototype.stop = function () { }; ZongJi.prototype.pause = function () { - this.connection.pause(); + if (!this.stopped) { + this.connection.pause(); + } }; ZongJi.prototype.resume = function () { - this.connection.resume(); + if (!this.stopped) { + this.connection.resume(); + } }; // It includes every events by default. diff --git a/test/events.js b/test/events.js index 7841057..5105d71 100644 --- a/test/events.js +++ b/test/events.js @@ -440,3 +440,77 @@ tap.test('With many columns', (test) => { ); }); }); + +tap.test('Pause/Resume Binlog', (test) => { + const TEST_TABLE = 'pause_resume_test_table'; + + test.test(`prepare table ${TEST_TABLE}`, (test) => { + testDb.execute([`DROP TABLE IF EXISTS ${TEST_TABLE}`, `CREATE TABLE ${TEST_TABLE} (col INT UNSIGNED)`], (err) => { + if (err) { + return test.fail(err); + } + + test.end(); + }); + }); + + tap.test('Pausing binlog stops events', (test) => { + const events = []; + const zongji = new ZongJi(settings.connection); + test.teardown(() => zongji.stop()); + + zongji.on('ready', () => { + zongji.pause(); + paused = true; + testDb.execute( + [ + `INSERT INTO ${TEST_TABLE} (col) + VALUES (14)` + ], + (err) => { + if (err) { + return test.fail(err); + } + } + ); + setTimeout(() => { + paused = false; + zongji.resume(); + }, 20); + }); + + zongji.start({ + startAtEnd: true, + serverId: testDb.serverId(), + includeEvents: ['tablemap', 'writerows'] + }); + + let paused = false; + zongji.on('binlog', (evt) => { + if (paused) { + // We don't expect any events while paused + test.fail(); + return; + } + + events.push(evt); + if (events.length == 2) { + expectEvents( + test, + events, + [ + tableMapEvent(TEST_TABLE), + { + _type: 'WriteRows', + _checkTableMap: checkTableMatches(TEST_TABLE), + rows: [{ col: 14 }] + } + ], + 1, + () => test.end() + ); + } + }); + }); + test.end(); +}); From 0804f1dc6c567f13c474f13dedc95dcc68291b14 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 28 May 2025 12:52:53 +0200 Subject: [PATCH 26/26] Fixed test setup --- test/events.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/events.js b/test/events.js index 5105d71..630ebf1 100644 --- a/test/events.js +++ b/test/events.js @@ -454,10 +454,11 @@ tap.test('Pause/Resume Binlog', (test) => { }); }); - tap.test('Pausing binlog stops events', (test) => { + test.test('Pausing binlog stops events', (test) => { const events = []; const zongji = new ZongJi(settings.connection); test.teardown(() => zongji.stop()); + let paused = false; zongji.on('ready', () => { zongji.pause(); @@ -485,7 +486,6 @@ tap.test('Pause/Resume Binlog', (test) => { includeEvents: ['tablemap', 'writerows'] }); - let paused = false; zongji.on('binlog', (evt) => { if (paused) { // We don't expect any events while paused