From 321a6b50fe06e022d06976d0dc0644f7c77cb9a0 Mon Sep 17 00:00:00 2001 From: DeveloperChaseLewis Date: Mon, 21 Apr 2025 17:12:12 -0500 Subject: [PATCH 1/5] Fix various issues causing stack overflow if large table is requested. Also add a slightly more optimal code path for initial subscriptions. --- packages/sdk/src/db_connection_impl.ts | 6 +-- packages/sdk/src/operations_map.ts | 63 -------------------------- packages/sdk/src/table_cache.ts | 60 ++++++++++++++---------- 3 files changed, 37 insertions(+), 92 deletions(-) delete mode 100644 packages/sdk/src/operations_map.ts diff --git a/packages/sdk/src/db_connection_impl.ts b/packages/sdk/src/db_connection_impl.ts index 640399b..1294342 100644 --- a/packages/sdk/src/db_connection_impl.ts +++ b/packages/sdk/src/db_connection_impl.ts @@ -519,15 +519,13 @@ export class DbConnectionImpl< tableUpdates: TableUpdate[], eventContext: EventContextInterface ): PendingCallback[] { - const pendingCallbacks: PendingCallback[] = []; + let pendingCallbacks: PendingCallback[] = []; for (let tableUpdate of tableUpdates) { // Get table information for the table being updated const tableName = tableUpdate.tableName; const tableTypeInfo = this.#remoteModule.tables[tableName]!; const table = this.clientCache.getOrCreateTable(tableTypeInfo); - pendingCallbacks.push( - ...table.applyOperations(tableUpdate.operations, eventContext) - ); + pendingCallbacks = pendingCallbacks.concat(table.applyOperations(tableUpdate.operations, eventContext)); } return pendingCallbacks; } diff --git a/packages/sdk/src/operations_map.ts b/packages/sdk/src/operations_map.ts deleted file mode 100644 index 6ba22f4..0000000 --- a/packages/sdk/src/operations_map.ts +++ /dev/null @@ -1,63 +0,0 @@ -export default class OperationsMap { - #items: { key: K; value: V }[] = []; - - #isEqual(a: K, b: K): boolean { - if (a && typeof a === 'object' && 'isEqual' in a) { - return (a as any).isEqual(b); - } - return a === b; - } - - set(key: K, value: V): void { - const existingIndex = this.#items.findIndex(({ key: k }) => - this.#isEqual(k, key) - ); - if (existingIndex > -1) { - this.#items[existingIndex].value = value; - } else { - this.#items.push({ key, value }); - } - } - - get(key: K): V | undefined { - const item = this.#items.find(({ key: k }) => this.#isEqual(k, key)); - return item ? item.value : undefined; - } - - delete(key: K): boolean { - const existingIndex = this.#items.findIndex(({ key: k }) => - this.#isEqual(k, key) - ); - if (existingIndex > -1) { - this.#items.splice(existingIndex, 1); - return true; - } - return false; - } - - has(key: K): boolean { - return this.#items.some(({ key: k }) => this.#isEqual(k, key)); - } - - values(): Array { - return this.#items.map(i => i.value); - } - - entries(): Array<{ key: K; value: V }> { - return this.#items; - } - - [Symbol.iterator](): Iterator<{ key: K; value: V }> { - let index = 0; - const items = this.#items; - return { - next(): IteratorResult<{ key: K; value: V }> { - if (index < items.length) { - return { value: items[index++], done: false }; - } else { - return { value: null, done: true }; - } - }, - }; - } -} diff --git a/packages/sdk/src/table_cache.ts b/packages/sdk/src/table_cache.ts index d23968c..e7a538f 100644 --- a/packages/sdk/src/table_cache.ts +++ b/packages/sdk/src/table_cache.ts @@ -1,5 +1,4 @@ import { EventEmitter } from './event_emitter.ts'; -import OperationsMap from './operations_map.ts'; import type { TableRuntimeTypeInfo } from './spacetime_module.ts'; import { type EventContextInterface } from './db_connection_impl.ts'; @@ -62,8 +61,9 @@ export class TableCache { const pendingCallbacks: PendingCallback[] = []; if (this.tableTypeInfo.primaryKey !== undefined) { const primaryKey = this.tableTypeInfo.primaryKey; - const insertMap = new OperationsMap(); - const deleteMap = new OperationsMap(); + const insertMap = new Map(); + const deleteMap = new Map(); + for (const op of operations) { if (op.type === 'insert') { const [_, prevCount] = insertMap.get(op.row[primaryKey]) || [op, 0]; @@ -73,34 +73,44 @@ export class TableCache { deleteMap.set(op.row[primaryKey], [op, prevCount + 1]); } } - for (const { - key: primaryKey, - value: [insertOp, refCount], - } of insertMap) { - const deleteEntry = deleteMap.get(primaryKey); - if (deleteEntry) { - const [deleteOp, deleteCount] = deleteEntry; - // In most cases the refCountDelta will be either 0 or refCount, but if - // an update moves a row in or out of the result set of different queries, then - // other deltas are possible. - const refCountDelta = refCount - deleteCount; - const maybeCb = this.update(ctx, insertOp, deleteOp, refCountDelta); - if (maybeCb) { - pendingCallbacks.push(maybeCb); + + if(deleteMap.size > 0) + { + for (const [primaryKey,[insertOp,refCount]] of insertMap.entries()) { + const deleteEntry = deleteMap.get(primaryKey); + if (deleteEntry) { + const [deleteOp, deleteCount] = deleteEntry; + // In most cases the refCountDelta will be either 0 or refCount, but if + // an update moves a row in or out of the result set of different queries, then + // other deltas are possible. + const refCountDelta = refCount - deleteCount; + const maybeCb = this.update(ctx, insertOp, deleteOp, refCountDelta); + if (maybeCb) { + pendingCallbacks.push(maybeCb); + } + deleteMap.delete(primaryKey); + } else { + const maybeCb = this.insert(ctx, insertOp, refCount); + if (maybeCb) { + pendingCallbacks.push(maybeCb); + } } - deleteMap.delete(primaryKey); - } else { - const maybeCb = this.insert(ctx, insertOp, refCount); + } + for (const [deleteOp, refCount] of deleteMap.values()) { + const maybeCb = this.delete(ctx, deleteOp, refCount); if (maybeCb) { pendingCallbacks.push(maybeCb); } } } - for (const [deleteOp, refCount] of deleteMap.values()) { - const maybeCb = this.delete(ctx, deleteOp, refCount); - if (maybeCb) { - pendingCallbacks.push(maybeCb); - } + else + { + for (const [insertOp,refCount] of insertMap.values()) { + const maybeCb = this.insert(ctx, insertOp, refCount); + if (maybeCb) { + pendingCallbacks.push(maybeCb); + } + } } } else { for (const op of operations) { From 717319f8fec16d8392d9d0194fef698b1f7e1ffb Mon Sep 17 00:00:00 2001 From: DeveloperChaseLewis Date: Mon, 21 Apr 2025 17:47:54 -0500 Subject: [PATCH 2/5] format changes --- examples/quickstart-chat/src/index.css | 10 ++--- packages/sdk/package.json | 6 ++- packages/sdk/src/db_connection_impl.ts | 4 +- packages/sdk/src/table_cache.ts | 54 ++++++++++---------------- 4 files changed, 34 insertions(+), 40 deletions(-) diff --git a/examples/quickstart-chat/src/index.css b/examples/quickstart-chat/src/index.css index 9390800..12a6ed0 100644 --- a/examples/quickstart-chat/src/index.css +++ b/examples/quickstart-chat/src/index.css @@ -32,16 +32,16 @@ body, } body { - font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', - 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', - sans-serif; + font-family: + -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu', + 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', sans-serif; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; } code { - font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', - monospace; + font-family: + source-code-pro, Menlo, Monaco, Consolas, 'Courier New', monospace; } /* ----- Buttons ----- */ diff --git a/packages/sdk/package.json b/packages/sdk/package.json index c451a66..50a7a11 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -43,7 +43,11 @@ }, "devDependencies": { "@clockworklabs/test-app": "file:../test-app", - "tsup": "^8.1.0", + "tsup": "^8.4.0", "undici": "^6.19.2" + }, + "dependencies": { + "typescript": "^5.8.3", + "vitest": "^3.1.2" } } diff --git a/packages/sdk/src/db_connection_impl.ts b/packages/sdk/src/db_connection_impl.ts index 1294342..ba7e240 100644 --- a/packages/sdk/src/db_connection_impl.ts +++ b/packages/sdk/src/db_connection_impl.ts @@ -525,7 +525,9 @@ export class DbConnectionImpl< const tableName = tableUpdate.tableName; const tableTypeInfo = this.#remoteModule.tables[tableName]!; const table = this.clientCache.getOrCreateTable(tableTypeInfo); - pendingCallbacks = pendingCallbacks.concat(table.applyOperations(tableUpdate.operations, eventContext)); + pendingCallbacks = pendingCallbacks.concat( + table.applyOperations(tableUpdate.operations, eventContext) + ); } return pendingCallbacks; } diff --git a/packages/sdk/src/table_cache.ts b/packages/sdk/src/table_cache.ts index e7a538f..2949c59 100644 --- a/packages/sdk/src/table_cache.ts +++ b/packages/sdk/src/table_cache.ts @@ -61,8 +61,8 @@ export class TableCache { const pendingCallbacks: PendingCallback[] = []; if (this.tableTypeInfo.primaryKey !== undefined) { const primaryKey = this.tableTypeInfo.primaryKey; - const insertMap = new Map(); - const deleteMap = new Map(); + const insertMap = new Map(); + const deleteMap = new Map(); for (const op of operations) { if (op.type === 'insert') { @@ -74,43 +74,31 @@ export class TableCache { } } - if(deleteMap.size > 0) - { - for (const [primaryKey,[insertOp,refCount]] of insertMap.entries()) { - const deleteEntry = deleteMap.get(primaryKey); - if (deleteEntry) { - const [deleteOp, deleteCount] = deleteEntry; - // In most cases the refCountDelta will be either 0 or refCount, but if - // an update moves a row in or out of the result set of different queries, then - // other deltas are possible. - const refCountDelta = refCount - deleteCount; - const maybeCb = this.update(ctx, insertOp, deleteOp, refCountDelta); - if (maybeCb) { - pendingCallbacks.push(maybeCb); - } - deleteMap.delete(primaryKey); - } else { - const maybeCb = this.insert(ctx, insertOp, refCount); - if (maybeCb) { - pendingCallbacks.push(maybeCb); - } + for (const [primaryKey, [insertOp, refCount]] of insertMap.entries()) { + const deleteEntry = deleteMap.get(primaryKey); + if (deleteEntry) { + const [deleteOp, deleteCount] = deleteEntry; + // In most cases the refCountDelta will be either 0 or refCount, but if + // an update moves a row in or out of the result set of different queries, then + // other deltas are possible. + const refCountDelta = refCount - deleteCount; + const maybeCb = this.update(ctx, insertOp, deleteOp, refCountDelta); + if (maybeCb) { + pendingCallbacks.push(maybeCb); } - } - for (const [deleteOp, refCount] of deleteMap.values()) { - const maybeCb = this.delete(ctx, deleteOp, refCount); + deleteMap.delete(primaryKey); + } else { + const maybeCb = this.insert(ctx, insertOp, refCount); if (maybeCb) { pendingCallbacks.push(maybeCb); } } } - else - { - for (const [insertOp,refCount] of insertMap.values()) { - const maybeCb = this.insert(ctx, insertOp, refCount); - if (maybeCb) { - pendingCallbacks.push(maybeCb); - } - } + for (const [deleteOp, refCount] of deleteMap.values()) { + const maybeCb = this.delete(ctx, deleteOp, refCount); + if (maybeCb) { + pendingCallbacks.push(maybeCb); + } } } else { for (const op of operations) { From 38484cd67d67032b5a1e2088643bb18b599d8db2 Mon Sep 17 00:00:00 2001 From: DeveloperChaseLewis Date: Mon, 21 Apr 2025 18:09:18 -0500 Subject: [PATCH 3/5] update to handle complex keys --- packages/sdk/src/connection_id.ts | 4 ++++ packages/sdk/src/identity.ts | 4 ++++ packages/sdk/src/table_cache.ts | 13 +++++++++---- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/sdk/src/connection_id.ts b/packages/sdk/src/connection_id.ts index c8d9501..e96b5e1 100644 --- a/packages/sdk/src/connection_id.ts +++ b/packages/sdk/src/connection_id.ts @@ -47,6 +47,10 @@ export class ConnectionId { return this.data == other.data; } + toPrimaryKey(): bigint { + return this.data; + } + /** * Print the connection ID as a hexadecimal string. */ diff --git a/packages/sdk/src/identity.ts b/packages/sdk/src/identity.ts index 9ce25dc..d2df27b 100644 --- a/packages/sdk/src/identity.ts +++ b/packages/sdk/src/identity.ts @@ -37,6 +37,10 @@ export class Identity { return u256ToHexString(this.data); } + toPrimaryKey(): string { + return this.toHexString(); + } + /** * Convert the address to a Uint8Array. */ diff --git a/packages/sdk/src/table_cache.ts b/packages/sdk/src/table_cache.ts index 2949c59..2fc1321 100644 --- a/packages/sdk/src/table_cache.ts +++ b/packages/sdk/src/table_cache.ts @@ -65,12 +65,17 @@ export class TableCache { const deleteMap = new Map(); for (const op of operations) { + let key = op.row[primaryKey]; + if (typeof key === 'object' && 'toPrimaryKey' in key) { + key = key.toPrimaryKey(); + } + if (op.type === 'insert') { - const [_, prevCount] = insertMap.get(op.row[primaryKey]) || [op, 0]; - insertMap.set(op.row[primaryKey], [op, prevCount + 1]); + const [_, prevCount] = insertMap.get(key) || [op, 0]; + insertMap.set(key, [op, prevCount + 1]); } else { - const [_, prevCount] = deleteMap.get(op.row[primaryKey]) || [op, 0]; - deleteMap.set(op.row[primaryKey], [op, prevCount + 1]); + const [_, prevCount] = deleteMap.get(key) || [op, 0]; + deleteMap.set(key, [op, prevCount + 1]); } } From 5474326dd476a5df638b57787ef7aab16c5c113c Mon Sep 17 00:00:00 2001 From: DeveloperChaseLewis Date: Mon, 21 Apr 2025 18:17:05 -0500 Subject: [PATCH 4/5] remove accidental change to package.json --- packages/sdk/package.json | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 50a7a11..c451a66 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -43,11 +43,7 @@ }, "devDependencies": { "@clockworklabs/test-app": "file:../test-app", - "tsup": "^8.4.0", + "tsup": "^8.1.0", "undici": "^6.19.2" - }, - "dependencies": { - "typescript": "^5.8.3", - "vitest": "^3.1.2" } } From 621fb2ac37459481353ac6221fe7501bb57b9066 Mon Sep 17 00:00:00 2001 From: DeveloperChaseLewis Date: Mon, 21 Apr 2025 22:02:51 -0500 Subject: [PATCH 5/5] optimization when there are no deletes in the table cache operation list. --- packages/sdk/src/table_cache.ts | 46 ++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/packages/sdk/src/table_cache.ts b/packages/sdk/src/table_cache.ts index 2fc1321..88c5318 100644 --- a/packages/sdk/src/table_cache.ts +++ b/packages/sdk/src/table_cache.ts @@ -60,10 +60,10 @@ export class TableCache { ): PendingCallback[] => { const pendingCallbacks: PendingCallback[] = []; if (this.tableTypeInfo.primaryKey !== undefined) { + let hasDelete = false; const primaryKey = this.tableTypeInfo.primaryKey; const insertMap = new Map(); const deleteMap = new Map(); - for (const op of operations) { let key = op.row[primaryKey]; if (typeof key === 'object' && 'toPrimaryKey' in key) { @@ -74,37 +74,47 @@ export class TableCache { const [_, prevCount] = insertMap.get(key) || [op, 0]; insertMap.set(key, [op, prevCount + 1]); } else { + hasDelete = true; const [_, prevCount] = deleteMap.get(key) || [op, 0]; deleteMap.set(key, [op, prevCount + 1]); } } - for (const [primaryKey, [insertOp, refCount]] of insertMap.entries()) { - const deleteEntry = deleteMap.get(primaryKey); - if (deleteEntry) { - const [deleteOp, deleteCount] = deleteEntry; - // In most cases the refCountDelta will be either 0 or refCount, but if - // an update moves a row in or out of the result set of different queries, then - // other deltas are possible. - const refCountDelta = refCount - deleteCount; - const maybeCb = this.update(ctx, insertOp, deleteOp, refCountDelta); + if (hasDelete) { + for (const [primaryKey, [insertOp, refCount]] of insertMap.entries()) { + const deleteEntry = deleteMap.get(primaryKey); + if (deleteEntry) { + const [deleteOp, deleteCount] = deleteEntry; + // In most cases the refCountDelta will be either 0 or refCount, but if + // an update moves a row in or out of the result set of different queries, then + // other deltas are possible. + const refCountDelta = refCount - deleteCount; + const maybeCb = this.update(ctx, insertOp, deleteOp, refCountDelta); + if (maybeCb) { + pendingCallbacks.push(maybeCb); + } + deleteMap.delete(primaryKey); + } else { + const maybeCb = this.insert(ctx, insertOp, refCount); + if (maybeCb) { + pendingCallbacks.push(maybeCb); + } + } + } + for (const [deleteOp, refCount] of deleteMap.values()) { + const maybeCb = this.delete(ctx, deleteOp, refCount); if (maybeCb) { pendingCallbacks.push(maybeCb); } - deleteMap.delete(primaryKey); - } else { + } + } else { + for (const [insertOp, refCount] of insertMap.values()) { const maybeCb = this.insert(ctx, insertOp, refCount); if (maybeCb) { pendingCallbacks.push(maybeCb); } } } - for (const [deleteOp, refCount] of deleteMap.values()) { - const maybeCb = this.delete(ctx, deleteOp, refCount); - if (maybeCb) { - pendingCallbacks.push(maybeCb); - } - } } else { for (const op of operations) { if (op.type === 'insert') {