From b264ca2ab69ac154cde41bacbb7a6059507207e7 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 11 Jan 2025 23:02:39 +0000 Subject: [PATCH 001/281] nats: proof of concept experiments --- src/packages/frontend/client/client.ts | 1 + src/packages/frontend/client/nats.ts | 13 ++++++++++ src/packages/frontend/package.json | 1 + src/packages/hub/hub.ts | 3 +++ src/packages/hub/package.json | 1 + src/packages/hub/servers/nats.ts | 19 ++++++++++++++ src/packages/pnpm-lock.yaml | 34 ++++++++++++++++++++++++++ 7 files changed, 72 insertions(+) create mode 100644 src/packages/frontend/client/nats.ts create mode 100644 src/packages/hub/servers/nats.ts diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 061e5e685b..eedee8da65 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -28,6 +28,7 @@ import { version } from "@cocalc/util/smc-version"; import { start_metrics } from "../prom-client"; import { setup_global_cocalc } from "./console"; import { Query } from "@cocalc/sync/table"; +import "./nats"; import debug from "debug"; // This DEBUG variable comes from webpack: diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts new file mode 100644 index 0000000000..ec3dbd0703 --- /dev/null +++ b/src/packages/frontend/client/nats.ts @@ -0,0 +1,13 @@ +import { connect, StringCodec } from "nats.ws"; + +export async function init() { + console.log("connecting..."); + const nc = await connect({ + servers: ["ws://localhost:5004"], + }); + console.log(`connected to ${nc.getServer()}`); + return nc; +} +const sc = StringCodec(); + +// window.x = { init, sc }; diff --git a/src/packages/frontend/package.json b/src/packages/frontend/package.json index 446da3bc0e..f93bd99bcf 100644 --- a/src/packages/frontend/package.json +++ b/src/packages/frontend/package.json @@ -127,6 +127,7 @@ "md5": "^2", "memoize-one": "^5.1.1", "mermaid": "^11.3.0", + "nats.ws": "^1.30.0", "node-forge": "^1.0.0", "nyc": "^15.1.0", "octicons": "^3.5.0", diff --git a/src/packages/hub/hub.ts b/src/packages/hub/hub.ts index 7b690468c7..fed6f4c8f7 100644 --- a/src/packages/hub/hub.ts +++ b/src/packages/hub/hub.ts @@ -41,6 +41,7 @@ import { start as startHubRegister } from "./hub_register"; import { getLogger } from "./logger"; import initDatabase, { database } from "./servers/database"; import initExpressApp from "./servers/express-app"; +import initNats from "./servers/nats"; import initHttpRedirect from "./servers/http-redirect"; import initPrimus from "./servers/primus"; import initVersionServer from "./servers/version"; @@ -228,6 +229,8 @@ async function startServer(): Promise { process.env["NODE_ENV"] == "development", }); + initNats(); + // The express app create via initExpressApp above **assumes** that init_passport is done // or complains a lot. This is obviously not really necessary, but we leave it for now. await callback2(init_passport, { diff --git a/src/packages/hub/package.json b/src/packages/hub/package.json index 8f0b28c323..78182fa722 100644 --- a/src/packages/hub/package.json +++ b/src/packages/hub/package.json @@ -47,6 +47,7 @@ "mime": "^1.3.4", "mkdirp": "^1.0.4", "ms": "2.1.2", + "nats": "^2.29.1", "next": "14.2.22", "nyc": "^15.1.0", "parse-domain": "^5.0.0", diff --git a/src/packages/hub/servers/nats.ts b/src/packages/hub/servers/nats.ts new file mode 100644 index 0000000000..14dc9a2788 --- /dev/null +++ b/src/packages/hub/servers/nats.ts @@ -0,0 +1,19 @@ +import { connect, StringCodec } from "nats"; + +export default async function initNats() { + console.log("initializing nats echo server"); + const nc = await connect(); + console.log(`connected to ${nc.getServer()}`); + const sc = StringCodec(); + + const sub = nc.subscribe("echo"); + const handle = (msg) => { + const data = sc.decode(msg.data); + console.log(`Received: ${data}`); + msg.respond(sc.encode("echo from HUB - " + data)); + }; + + for await (const msg of sub) { + handle(msg); + } +} diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 32aa61dc88..173c8f6943 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -508,6 +508,9 @@ importers: mermaid: specifier: ^11.3.0 version: 11.3.0 + nats.ws: + specifier: ^1.30.0 + version: 1.30.0 node-forge: specifier: ^1.0.0 version: 1.3.1 @@ -824,6 +827,9 @@ importers: ms: specifier: 2.1.2 version: 2.1.2 + nats: + specifier: ^2.29.1 + version: 2.29.1 next: specifier: 14.2.22 version: 14.2.22(@babel/core@7.25.8)(@opentelemetry/api@1.9.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.81.0) @@ -8880,6 +8886,13 @@ packages: native-promise-only@0.8.1: resolution: {integrity: sha512-zkVhZUA3y8mbz652WrL5x0fB0ehrBkulWT3TomAQ9iDtyXZvzKeEA6GPxAItBYeNYl5yngKRX612qHOhvMkDeg==} + nats.ws@1.30.0: + resolution: {integrity: sha512-AjW86u40BcDiYUHBUTBwDTkZDehZB3bMLw7R1K15eFt8U3YUCxhUHhcL57ciEqUiaNdSdYIkJsnE8BFxY/RuDg==} + + nats@2.29.1: + resolution: {integrity: sha512-OHVsxrQCITTdMKG3So0jhtnBd5jS2u1xpS91UCws7VklsaCbctwg5vT/8lYpVldPW0x3aHGF8uuAoMfCoJy7Sg==} + engines: {node: '>= 14.0.0'} + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -8946,6 +8959,10 @@ packages: nise@1.5.3: resolution: {integrity: sha512-Ymbac/94xeIrMf59REBPOv0thr+CJVFMhrlAkW/gjCIE58BGQdCj0x7KRCb3yz+Ga2Rz3E9XXSvUyyxqqhjQAQ==} + nkeys.js@1.1.0: + resolution: {integrity: sha512-tB/a0shZL5UZWSwsoeyqfTszONTt4k2YS0tuQioMOD180+MbombYVgzDUYHlx+gejYK6rgf08n/2Df99WY0Sxg==} + engines: {node: '>=10.0.0'} + no-case@3.0.4: resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} @@ -11370,6 +11387,9 @@ packages: tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + tweetnacl@1.0.3: + resolution: {integrity: sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -20221,6 +20241,14 @@ snapshots: native-promise-only@0.8.1: {} + nats.ws@1.30.0: + optionalDependencies: + nkeys.js: 1.1.0 + + nats@2.29.1: + dependencies: + nkeys.js: 1.1.0 + natural-compare@1.4.0: {} ncp@2.0.0: @@ -20336,6 +20364,10 @@ snapshots: lolex: 5.1.2 path-to-regexp: 1.9.0 + nkeys.js@1.1.0: + dependencies: + tweetnacl: 1.0.3 + no-case@3.0.4: dependencies: lower-case: 2.0.2 @@ -23251,6 +23283,8 @@ snapshots: dependencies: safe-buffer: 5.2.1 + tweetnacl@1.0.3: {} + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 From 20210f0b8627f81421d6c43b00ddabc1ed55c05b Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 12 Jan 2025 01:50:41 +0000 Subject: [PATCH 002/281] nats POC: proxying nats websocket server using node-proxy - it just works fine, even with nextjs around and a base path, etc.! :-) --- src/packages/frontend/client/client.ts | 5 ++- src/packages/frontend/client/nats.ts | 41 ++++++++++++++----- src/packages/hub/hub.ts | 2 +- src/packages/hub/proxy/handle-upgrade.ts | 10 +++-- src/packages/hub/servers/nats.ts | 51 +++++++++++++++++++++--- 5 files changed, 88 insertions(+), 21 deletions(-) diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index eedee8da65..5bd3e13ea1 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -22,13 +22,13 @@ import { SyncClient } from "@cocalc/sync/client/sync-client"; import { UsersClient } from "./users"; import { FileClient } from "./file"; import { TrackingClient } from "./tracking"; +import { NatsClient } from "./nats"; import { HubClient } from "./hub"; import { IdleClient } from "./idle"; import { version } from "@cocalc/util/smc-version"; import { start_metrics } from "../prom-client"; import { setup_global_cocalc } from "./console"; import { Query } from "@cocalc/sync/table"; -import "./nats"; import debug from "debug"; // This DEBUG variable comes from webpack: @@ -64,6 +64,7 @@ export interface WebappClient extends EventEmitter { users_client: UsersClient; file_client: FileClient; tracking_client: TrackingClient; + nats_client: NatsClient; hub_client: HubClient; idle_client: IdleClient; client: Client; @@ -143,6 +144,7 @@ class Client extends EventEmitter implements WebappClient { users_client: UsersClient; file_client: FileClient; tracking_client: TrackingClient; + nats_client: NatsClient; hub_client: HubClient; idle_client: IdleClient; client: Client; @@ -232,6 +234,7 @@ class Client extends EventEmitter implements WebappClient { new UsersClient(this.call.bind(this), this.async_call.bind(this)), ); this.tracking_client = bind_methods(new TrackingClient(this)); + this.nats_client = bind_methods(new NatsClient(this)); this.file_client = bind_methods(new FileClient(this.async_call.bind(this))); this.idle_client = bind_methods(new IdleClient(this)); diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index ec3dbd0703..3d760b0627 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -1,13 +1,34 @@ -import { connect, StringCodec } from "nats.ws"; +import * as nats from "nats.ws"; +import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; +import type { WebappClient } from "./client"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -export async function init() { - console.log("connecting..."); - const nc = await connect({ - servers: ["ws://localhost:5004"], +export class NatsClient { + /*private*/ client: WebappClient; + private sc: ReturnType; + private nc?: Awaited>; + // obviously just for learning: + public nats = nats; + + constructor(client: WebappClient) { + this.client = client; + this.sc = nats.StringCodec(); + } + + getConnection = reuseInFlight(async () => { + if (this.nc != null) { + return this.nc; + } + const server = `${location.protocol == "https:" ? "wss" : "ws"}://${location.host}${appBasePath}/nats`; + console.log(`connecting to ${server}...`); + this.nc = await nats.connect({ servers: [server] }); + console.log(`connected to ${server}`); + return this.nc; }); - console.log(`connected to ${nc.getServer()}`); - return nc; -} -const sc = StringCodec(); -// window.x = { init, sc }; + request = async (subject: string, data: string) => { + const c = await this.getConnection(); + const resp = await c.request(subject, this.sc.encode(data)); + return this.sc.decode(resp.data); + }; +} diff --git a/src/packages/hub/hub.ts b/src/packages/hub/hub.ts index fed6f4c8f7..4610c4207a 100644 --- a/src/packages/hub/hub.ts +++ b/src/packages/hub/hub.ts @@ -41,7 +41,7 @@ import { start as startHubRegister } from "./hub_register"; import { getLogger } from "./logger"; import initDatabase, { database } from "./servers/database"; import initExpressApp from "./servers/express-app"; -import initNats from "./servers/nats"; +import { initNats } from "./servers/nats"; import initHttpRedirect from "./servers/http-redirect"; import initPrimus from "./servers/primus"; import initVersionServer from "./servers/version"; diff --git a/src/packages/hub/proxy/handle-upgrade.ts b/src/packages/hub/proxy/handle-upgrade.ts index 2a5e5c38d6..acf67e41e2 100644 --- a/src/packages/hub/proxy/handle-upgrade.ts +++ b/src/packages/hub/proxy/handle-upgrade.ts @@ -3,12 +3,12 @@ import { createProxyServer } from "http-proxy"; import LRU from "lru-cache"; import { getEventListeners } from "node:events"; - import getLogger from "@cocalc/hub/logger"; import stripRememberMeCookie from "./strip-remember-me-cookie"; import { getTarget } from "./target"; import { stripBasePath } from "./util"; import { versionCheckFails } from "./version"; +import { proxyNatsWebsocket } from "@cocalc/hub/servers/nats"; const logger = getLogger("proxy:handle-upgrade"); @@ -32,6 +32,12 @@ export default function init( logger.silly(req.url, ...args); }; dbg("got upgrade request from url=", req.url); + const url = stripBasePath(req.url); + + if (url == "/nats") { + return proxyNatsWebsocket(req, socket, head); + } + if (!req.url.match(re)) { throw Error(`url=${req.url} does not support upgrade`); } @@ -44,8 +50,6 @@ export default function init( throw Error("client version check failed"); } - const url = stripBasePath(req.url); - let remember_me, api_key; if (req.headers["cookie"] != null) { let cookie; diff --git a/src/packages/hub/servers/nats.ts b/src/packages/hub/servers/nats.ts index 14dc9a2788..b2152fb9ba 100644 --- a/src/packages/hub/servers/nats.ts +++ b/src/packages/hub/servers/nats.ts @@ -1,16 +1,55 @@ +/* +Proof of concept NATS proxy. + +We assume there is a NATS server running on localhost with this configuration: + +# server.conf +websocket { + listen: "localhost:8443" + no_tls: true +} + +You could start this with + + nats-server -config server.conf + +*/ + import { connect, StringCodec } from "nats"; +import { createProxyServer } from "http-proxy"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("hub:nats"); + +const NATS = "ws://localhost:8443"; + +export async function initNats() { + logger.debug("initNats"); + // insecure just for fun test. + evalServer(); +} + +export async function proxyNatsWebsocket(req, socket, head) { + logger.debug("nats proxy -- handling a connection"); + const target = NATS; + const proxy = createProxyServer({ + ws: true, + target, + }); + proxy.ws(req, socket, head); +} -export default async function initNats() { - console.log("initializing nats echo server"); +async function evalServer() { + logger.debug("initializing nats echo server"); const nc = await connect(); - console.log(`connected to ${nc.getServer()}`); + logger.debug(`connected to ${nc.getServer()}`); const sc = StringCodec(); - const sub = nc.subscribe("echo"); + const sub = nc.subscribe("hub.eval"); const handle = (msg) => { const data = sc.decode(msg.data); - console.log(`Received: ${data}`); - msg.respond(sc.encode("echo from HUB - " + data)); + logger.debug("handling hub.eval", data); + msg.respond(sc.encode(`echo from HUB - ${eval(data)}`)); }; for await (const msg of sub) { From e60daf60ac4dec1a1946278439398af92a2817c1 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 12 Jan 2025 18:31:48 +0000 Subject: [PATCH 003/281] nats -- api/v2 via nats POC --- src/packages/database/settings/customize.ts | 199 ++++++++++---------- src/packages/frontend/client/nats.ts | 18 +- src/packages/hub/hub.ts | 4 +- src/packages/hub/servers/nats.ts | 30 +-- src/packages/next/pages/api/v2/customize.ts | 5 +- src/packages/pnpm-lock.yaml | 133 ++++++++----- src/packages/server/nats/api.ts | 96 ++++++++++ src/packages/server/nats/index.ts | 12 ++ src/packages/server/package.json | 7 +- 9 files changed, 318 insertions(+), 186 deletions(-) create mode 100644 src/packages/server/nats/api.ts create mode 100644 src/packages/server/nats/index.ts diff --git a/src/packages/database/settings/customize.ts b/src/packages/database/settings/customize.ts index 0eca745720..f5b649c0f2 100644 --- a/src/packages/database/settings/customize.ts +++ b/src/packages/database/settings/customize.ts @@ -11,6 +11,7 @@ import { import { Strategy } from "@cocalc/util/types/sso"; import { ServerSettings, getServerSettings } from "./server-settings"; import siteURL from "./site-url"; +import { copy_with } from "@cocalc/util/misc"; export interface Customize { siteName?: string; @@ -77,107 +78,107 @@ for a few seconds. let cachedSettings: ServerSettings | undefined = undefined; let cachedCustomize: Customize | undefined = undefined; -export default async function getCustomize(): Promise { +export default async function getCustomize( + fields?: string[], +): Promise { const [settings, strategies]: [ServerSettings, Strategy[]] = await Promise.all([getServerSettings(), getStrategies()]); - if (settings === cachedSettings && cachedCustomize != null) { - return cachedCustomize; + if (!(settings === cachedSettings && cachedCustomize != null)) { + cachedSettings = settings; + cachedCustomize = { + siteName: fallback(settings.site_name, "On Premises CoCalc"), + siteDescription: fallback( + settings.site_description, + "Collaborative Calculation using Python, Sage, R, Julia, and more.", + ), + + organizationName: settings.organization_name, + organizationEmail: settings.organization_email, + organizationURL: settings.organization_url, + termsOfServiceURL: settings.terms_of_service_url, + + helpEmail: settings.help_email, + contactEmail: fallback(settings.organization_email, settings.help_email), + + isCommercial: settings.commercial, + + kucalc: settings.kucalc, + sshGateway: settings.ssh_gateway, + sshGatewayDNS: settings.ssh_gateway_dns, + + anonymousSignup: settings.anonymous_signup, + anonymousSignupLicensedShares: settings.anonymous_signup_licensed_shares, + emailSignup: settings.email_signup, + accountCreationInstructions: settings.account_creation_email_instructions, + + logoSquareURL: settings.logo_square, + logoRectangularURL: settings.logo_rectangular, + splashImage: settings.splash_image, + + shareServer: !!settings.share_server, + + // additionally restrict showing landing pages only in cocalc.com-mode + landingPages: + !!settings.landing_pages && settings.kucalc === KUCALC_COCALC_COM, + + googleAnalytics: settings.google_analytics, + + indexInfo: settings.index_info_html, + indexTagline: settings.index_tagline, + imprint: settings.imprint, + policies: settings.policies, + support: settings.support, + + // Is important for invite emails, password reset, etc. (e.g., so we can construct a url to our site). + // This *can* start with http:// to explicitly use http instead of https, and can end + // in something like :3594 to indicate a port. + dns: settings.dns, + // siteURL is derived from settings.dns and the basePath -- it combines the dns, https:// + // and the basePath. It never ends in a slash. This is used in practice for + // things like invite emails, password reset, etc. + siteURL: await siteURL(settings.dns), + + zendesk: + settings.zendesk_token && + settings.zendesk_username && + settings.zendesk_uri, + + // obviously only the public key here! + stripePublishableKey: settings.stripe_publishable_key, + + // obviously only the public key here too! + reCaptchaKey: settings.re_captcha_v3_publishable_key, + + // a sandbox project + sandboxProjectId: settings.sandbox_project_id, + sandboxProjectsEnabled: settings.sandbox_projects_enabled, + + // true if openai integration is enabled -- this impacts the UI only, and can be + // turned on and off independently of whether there is an api key set. + openaiEnabled: settings.openai_enabled, + // same for google vertex (exposed as gemini), and others + googleVertexaiEnabled: settings.google_vertexai_enabled, + mistralEnabled: settings.mistral_enabled, + anthropicEnabled: settings.anthropic_enabled, + ollamaEnabled: settings.ollama_enabled, + + neuralSearchEnabled: settings.neural_search_enabled, + + // if extra Jupyter API functionality for sandboxed ephemeral code execution is available. + jupyterApiEnabled: settings.jupyter_api_enabled, + + computeServersEnabled: settings.compute_servers_enabled, + cloudFilesystemsEnabled: settings.cloud_filesystems_enabled, + + // GitHub proxy project + githubProjectId: settings.github_project_id, + + // public info about SSO strategies + strategies, + + verifyEmailAddresses: settings.verify_emails && settings.email_enabled, + }; } - cachedSettings = settings; - cachedCustomize = { - siteName: fallback(settings.site_name, "On Premises CoCalc"), - siteDescription: fallback( - settings.site_description, - "Collaborative Calculation using Python, Sage, R, Julia, and more.", - ), - - organizationName: settings.organization_name, - organizationEmail: settings.organization_email, - organizationURL: settings.organization_url, - termsOfServiceURL: settings.terms_of_service_url, - - helpEmail: settings.help_email, - contactEmail: fallback(settings.organization_email, settings.help_email), - - isCommercial: settings.commercial, - - kucalc: settings.kucalc, - sshGateway: settings.ssh_gateway, - sshGatewayDNS: settings.ssh_gateway_dns, - - anonymousSignup: settings.anonymous_signup, - anonymousSignupLicensedShares: settings.anonymous_signup_licensed_shares, - emailSignup: settings.email_signup, - accountCreationInstructions: settings.account_creation_email_instructions, - - logoSquareURL: settings.logo_square, - logoRectangularURL: settings.logo_rectangular, - splashImage: settings.splash_image, - - shareServer: !!settings.share_server, - - // additionally restrict showing landing pages only in cocalc.com-mode - landingPages: - !!settings.landing_pages && settings.kucalc === KUCALC_COCALC_COM, - - googleAnalytics: settings.google_analytics, - - indexInfo: settings.index_info_html, - indexTagline: settings.index_tagline, - imprint: settings.imprint, - policies: settings.policies, - support: settings.support, - - // Is important for invite emails, password reset, etc. (e.g., so we can construct a url to our site). - // This *can* start with http:// to explicitly use http instead of https, and can end - // in something like :3594 to indicate a port. - dns: settings.dns, - // siteURL is derived from settings.dns and the basePath -- it combines the dns, https:// - // and the basePath. It never ends in a slash. This is used in practice for - // things like invite emails, password reset, etc. - siteURL: await siteURL(settings.dns), - - zendesk: - settings.zendesk_token && - settings.zendesk_username && - settings.zendesk_uri, - - // obviously only the public key here! - stripePublishableKey: settings.stripe_publishable_key, - - // obviously only the public key here too! - reCaptchaKey: settings.re_captcha_v3_publishable_key, - - // a sandbox project - sandboxProjectId: settings.sandbox_project_id, - sandboxProjectsEnabled: settings.sandbox_projects_enabled, - - // true if openai integration is enabled -- this impacts the UI only, and can be - // turned on and off independently of whether there is an api key set. - openaiEnabled: settings.openai_enabled, - // same for google vertex (exposed as gemini), and others - googleVertexaiEnabled: settings.google_vertexai_enabled, - mistralEnabled: settings.mistral_enabled, - anthropicEnabled: settings.anthropic_enabled, - ollamaEnabled: settings.ollama_enabled, - - neuralSearchEnabled: settings.neural_search_enabled, - - // if extra Jupyter API functionality for sandboxed ephemeral code execution is available. - jupyterApiEnabled: settings.jupyter_api_enabled, - - computeServersEnabled: settings.compute_servers_enabled, - cloudFilesystemsEnabled: settings.cloud_filesystems_enabled, - - // GitHub proxy project - githubProjectId: settings.github_project_id, - - // public info about SSO strategies - strategies, - - verifyEmailAddresses: settings.verify_emails && settings.email_enabled, - }; - - return cachedCustomize; + return fields ? copy_with(cachedCustomize, fields) : cachedCustomize; } diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 3d760b0627..a5aabb4367 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -5,14 +5,14 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; export class NatsClient { /*private*/ client: WebappClient; - private sc: ReturnType; + private sc = nats.StringCodec(); + private jc = nats.JSONCodec(); private nc?: Awaited>; // obviously just for learning: public nats = nats; constructor(client: WebappClient) { this.client = client; - this.sc = nats.StringCodec(); } getConnection = reuseInFlight(async () => { @@ -31,4 +31,18 @@ export class NatsClient { const resp = await c.request(subject, this.sc.encode(data)); return this.sc.decode(resp.data); }; + + api = async (endpoint: string, params: object) => { + const c = await this.getConnection(); + const resp = await c.request( + "api.v2", + // obviously passing account_id is temporary -- need to use JWT. + this.jc.encode({ + endpoint, + __account_id: this.client.account_id, + ...params, + }), + ); + return this.jc.decode(resp.data); + }; } diff --git a/src/packages/hub/hub.ts b/src/packages/hub/hub.ts index 4610c4207a..e36585b2ff 100644 --- a/src/packages/hub/hub.ts +++ b/src/packages/hub/hub.ts @@ -41,7 +41,7 @@ import { start as startHubRegister } from "./hub_register"; import { getLogger } from "./logger"; import initDatabase, { database } from "./servers/database"; import initExpressApp from "./servers/express-app"; -import { initNats } from "./servers/nats"; +//import initNatsServer from "@cocalc/server/nats"; import initHttpRedirect from "./servers/http-redirect"; import initPrimus from "./servers/primus"; import initVersionServer from "./servers/version"; @@ -229,7 +229,7 @@ async function startServer(): Promise { process.env["NODE_ENV"] == "development", }); - initNats(); + //initNatsServer(); // The express app create via initExpressApp above **assumes** that init_passport is done // or complains a lot. This is obviously not really necessary, but we leave it for now. diff --git a/src/packages/hub/servers/nats.ts b/src/packages/hub/servers/nats.ts index b2152fb9ba..aaa35f61b5 100644 --- a/src/packages/hub/servers/nats.ts +++ b/src/packages/hub/servers/nats.ts @@ -15,44 +15,20 @@ You could start this with */ -import { connect, StringCodec } from "nats"; import { createProxyServer } from "http-proxy"; import getLogger from "@cocalc/backend/logger"; const logger = getLogger("hub:nats"); -const NATS = "ws://localhost:8443"; - -export async function initNats() { - logger.debug("initNats"); - // insecure just for fun test. - evalServer(); -} +// todo: move to database/server settings/etc.? +const NATS_WS = "ws://localhost:8443"; export async function proxyNatsWebsocket(req, socket, head) { logger.debug("nats proxy -- handling a connection"); - const target = NATS; + const target = NATS_WS; const proxy = createProxyServer({ ws: true, target, }); proxy.ws(req, socket, head); } - -async function evalServer() { - logger.debug("initializing nats echo server"); - const nc = await connect(); - logger.debug(`connected to ${nc.getServer()}`); - const sc = StringCodec(); - - const sub = nc.subscribe("hub.eval"); - const handle = (msg) => { - const data = sc.decode(msg.data); - logger.debug("handling hub.eval", data); - msg.respond(sc.encode(`echo from HUB - ${eval(data)}`)); - }; - - for await (const msg of sub) { - handle(msg); - } -} diff --git a/src/packages/next/pages/api/v2/customize.ts b/src/packages/next/pages/api/v2/customize.ts index ab131f91fb..e742087b4a 100644 --- a/src/packages/next/pages/api/v2/customize.ts +++ b/src/packages/next/pages/api/v2/customize.ts @@ -6,16 +6,13 @@ This calls something that is LRU cached on the server for a few seconds. */ import getCustomize from "@cocalc/database/settings/customize"; -import { copy_with } from "@cocalc/util/misc"; import getParams from "lib/api/get-params"; export default async function handle(req, res) { const { fields } = getParams(req); try { - const customize = await getCustomize(); - let result = fields ? copy_with(customize, fields) : customize; - res.json(result); + res.json(await getCustomize(fields)); } catch (err) { res.json({ error: `${err.message}` }); } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 173c8f6943..e3d2cce91b 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -188,7 +188,7 @@ importers: version: 8.7.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) immutable: specifier: ^4.3.0 version: 4.3.7 @@ -393,7 +393,7 @@ importers: version: 1.11.13 debug: specifier: ^4.3.4 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) direction: specifier: ^1.0.4 version: 1.0.4 @@ -790,7 +790,7 @@ importers: version: 2.8.5 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) escape-html: specifier: ^1.0.3 version: 1.0.3 @@ -968,7 +968,7 @@ importers: version: 8.7.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) enchannel-zmq-backend: specifier: ^9.1.23 version: 9.1.23(rxjs@7.8.1) @@ -1243,7 +1243,7 @@ importers: version: 3.0.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) diskusage: specifier: ^1.1.3 version: 1.2.0 @@ -1521,6 +1521,9 @@ importers: nanoid: specifier: ^3.3.8 version: 3.3.8 + nats: + specifier: ^2.29.1 + version: 2.29.1 node-zendesk: specifier: ^5.0.13 version: 5.0.13(encoding@0.1.13) @@ -1819,7 +1822,7 @@ importers: version: 3.0.0 debug: specifier: ^4.3.4 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) events: specifier: 3.3.0 version: 3.3.0 @@ -1871,7 +1874,7 @@ importers: version: 1.0.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) primus: specifier: ^8.0.9 version: 8.0.9 @@ -1951,7 +1954,7 @@ importers: version: 3.0.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) lodash: specifier: ^4.17.21 version: 4.17.21 @@ -1997,7 +2000,7 @@ importers: version: 1.11.13 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) decimal.js-light: specifier: ^2.5.1 version: 2.5.1 @@ -12272,10 +12275,10 @@ snapshots: '@babel/helpers': 7.25.6 '@babel/parser': 7.25.6 '@babel/template': 7.25.0 - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 '@babel/types': 7.25.6 convert-source-map: 2.0.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -12315,7 +12318,7 @@ snapshots: '@babel/traverse': 7.25.7 '@babel/types': 7.25.8 convert-source-map: 2.0.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -12335,7 +12338,7 @@ snapshots: '@babel/traverse': 7.25.9 '@babel/types': 7.26.0 convert-source-map: 2.0.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -12403,14 +12406,21 @@ snapshots: '@babel/helper-optimise-call-expression': 7.24.7 '@babel/helper-replace-supers': 7.25.0(@babel/core@7.25.2) '@babel/helper-skip-transparent-expression-wrappers': 7.24.7 - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 semver: 6.3.1 transitivePeerDependencies: - supports-color '@babel/helper-member-expression-to-functions@7.24.8': dependencies: - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 + '@babel/types': 7.25.6 + transitivePeerDependencies: + - supports-color + + '@babel/helper-module-imports@7.24.7': + dependencies: + '@babel/traverse': 7.25.6 '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -12450,10 +12460,10 @@ snapshots: '@babel/helper-module-transforms@7.25.2(@babel/core@7.25.2)': dependencies: '@babel/core': 7.25.2 - '@babel/helper-module-imports': 7.24.7(supports-color@9.4.0) - '@babel/helper-simple-access': 7.24.7(supports-color@9.4.0) + '@babel/helper-module-imports': 7.24.7 + '@babel/helper-simple-access': 7.24.7 '@babel/helper-validator-identifier': 7.24.7 - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 transitivePeerDependencies: - supports-color @@ -12488,7 +12498,14 @@ snapshots: '@babel/core': 7.25.2 '@babel/helper-member-expression-to-functions': 7.24.8 '@babel/helper-optimise-call-expression': 7.24.7 - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 + transitivePeerDependencies: + - supports-color + + '@babel/helper-simple-access@7.24.7': + dependencies: + '@babel/traverse': 7.25.6 + '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -12508,7 +12525,7 @@ snapshots: '@babel/helper-skip-transparent-expression-wrappers@7.24.7': dependencies: - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -12743,7 +12760,7 @@ snapshots: '@babel/core': 7.25.2 '@babel/helper-module-transforms': 7.25.2(@babel/core@7.25.2) '@babel/helper-plugin-utils': 7.24.8 - '@babel/helper-simple-access': 7.24.7(supports-color@9.4.0) + '@babel/helper-simple-access': 7.24.7 transitivePeerDependencies: - supports-color @@ -12797,6 +12814,18 @@ snapshots: '@babel/parser': 7.25.9 '@babel/types': 7.25.9 + '@babel/traverse@7.25.6': + dependencies: + '@babel/code-frame': 7.24.7 + '@babel/generator': 7.25.6 + '@babel/parser': 7.25.6 + '@babel/template': 7.25.0 + '@babel/types': 7.25.6 + debug: 4.3.7(supports-color@8.1.1) + globals: 11.12.0 + transitivePeerDependencies: + - supports-color + '@babel/traverse@7.25.6(supports-color@9.4.0)': dependencies: '@babel/code-frame': 7.24.7 @@ -12816,7 +12845,7 @@ snapshots: '@babel/parser': 7.25.8 '@babel/template': 7.25.9 '@babel/types': 7.25.8 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -12828,7 +12857,7 @@ snapshots: '@babel/parser': 7.26.2 '@babel/template': 7.25.9 '@babel/types': 7.26.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -12887,7 +12916,7 @@ snapshots: awaiting: 3.0.0 cheerio: 1.0.0-rc.12 csv-parse: 5.5.6 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -12901,7 +12930,7 @@ snapshots: '@cocalc/primus-responder@1.0.5': dependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) node-uuid: 1.4.8 transitivePeerDependencies: - supports-color @@ -12958,7 +12987,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -13123,7 +13152,7 @@ snapshots: '@humanwhocodes/config-array@0.13.0': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -13147,7 +13176,7 @@ snapshots: '@antfu/install-pkg': 0.4.1 '@antfu/utils': 0.7.10 '@iconify/types': 2.0.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) kolorist: 1.8.0 local-pkg: 0.5.0 mlly: 1.7.2 @@ -13276,7 +13305,7 @@ snapshots: istanbul-lib-coverage: 3.2.2 istanbul-lib-instrument: 6.0.3 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 4.0.1(supports-color@9.4.0) + istanbul-lib-source-maps: 4.0.1 istanbul-reports: 3.1.7 jest-message-util: 29.7.0 jest-util: 29.7.0 @@ -13909,7 +13938,7 @@ snapshots: '@types/xml-encryption': 1.2.4 '@types/xml2js': 0.4.14 '@xmldom/xmldom': 0.8.10 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) xml-crypto: 3.2.0 xml-encryption: 3.0.2 xml2js: 0.5.0 @@ -14955,7 +14984,7 @@ snapshots: '@typescript-eslint/type-utils': 6.21.0(eslint@8.57.1)(typescript@5.6.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.1)(typescript@5.6.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) eslint: 8.57.1 graphemer: 1.4.0 ignore: 5.3.1 @@ -14973,7 +15002,7 @@ snapshots: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.6.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) eslint: 8.57.1 optionalDependencies: typescript: 5.6.3 @@ -14989,7 +15018,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.6.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.1)(typescript@5.6.3) - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) eslint: 8.57.1 ts-api-utils: 1.3.0(typescript@5.6.3) optionalDependencies: @@ -15003,7 +15032,7 @@ snapshots: dependencies: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 @@ -15206,13 +15235,13 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color agent-base@7.1.1: dependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -17342,7 +17371,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -17704,11 +17733,11 @@ snapshots: follow-redirects@1.15.6(debug@4.3.7): optionalDependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) follow-redirects@1.15.9(debug@4.3.7): optionalDependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) font-atlas@2.1.0: dependencies: @@ -18454,7 +18483,7 @@ snapshots: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -18481,21 +18510,21 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.2: dependencies: agent-base: 7.1.1 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.4: dependencies: agent-base: 7.1.1 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -18518,7 +18547,7 @@ snapshots: '@types/tough-cookie': 4.0.5 axios: 1.7.4(debug@4.3.7) camelcase: 6.3.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) dotenv: 16.4.5 extend: 3.0.2 file-type: 16.5.4 @@ -18943,6 +18972,14 @@ snapshots: make-dir: 4.0.0 supports-color: 7.2.0 + istanbul-lib-source-maps@4.0.1: + dependencies: + debug: 4.3.7(supports-color@8.1.1) + istanbul-lib-coverage: 3.2.2 + source-map: 0.6.1 + transitivePeerDependencies: + - supports-color + istanbul-lib-source-maps@4.0.1(supports-color@9.4.0): dependencies: debug: 4.3.7(supports-color@9.4.0) @@ -20530,7 +20567,7 @@ snapshots: istanbul-lib-instrument: 4.0.3 istanbul-lib-processinfo: 2.0.3 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 4.0.1(supports-color@9.4.0) + istanbul-lib-source-maps: 4.0.1 istanbul-reports: 3.1.7 make-dir: 3.1.0 node-preload: 0.2.1 @@ -22712,7 +22749,7 @@ snapshots: spdy-transport@3.0.0: dependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) detect-node: 2.1.0 hpack.js: 2.1.6 obuf: 1.1.2 @@ -22723,7 +22760,7 @@ snapshots: spdy@4.0.2: dependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) handle-thing: 2.0.1 http-deceiver: 1.2.7 select-hose: 2.0.0 @@ -23854,7 +23891,7 @@ snapshots: dependencies: '@wwa/statvfs': 1.1.18 awaiting: 3.0.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) port-get: 1.0.4 ws: 8.18.0 transitivePeerDependencies: diff --git a/src/packages/server/nats/api.ts b/src/packages/server/nats/api.ts new file mode 100644 index 0000000000..57ccdaeca0 --- /dev/null +++ b/src/packages/server/nats/api.ts @@ -0,0 +1,96 @@ +/* +This is meant to be similar to the nexts pages http api/v2, but using NATS instead of HTTPS. + +To do development turn off for the hub, and run like this: + + echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node + +When you make changes, just restart it the above. All clients will instantly +use the new version after you restart, and there is no need to restart the hub +itself or any clients. + +To view all requests in realtime: + + nats sub api.v2 + +and if you want to also see the replies: + + nats sub api.v2 --match-replies +*/ + +import { JSONCodec } from "nats"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("server:nats"); + +const jc = JSONCodec(); + +export async function initAPI(nc) { + logger.debug("initAPI -- NATS api.v2 subject"); + const sub = nc.subscribe("api.v2"); + for await (const mesg of sub) { + handleApiRequest(mesg); + } +} + +async function handleApiRequest(mesg) { + const request = jc.decode(mesg.data) ?? {}; + let resp; + try { + // TODO: obviously user-provided account_id is no good! This is a POC. + const { endpoint, __account_id: account_id, ...params } = request as any; + logger.debug("handling api.v2 request:", { endpoint }); + resp = await getResponse(endpoint, account_id, params); + } catch (err) { + resp = { error: `${err}` }; + } + mesg.respond(jc.encode(resp)); +} + +import userQuery from "@cocalc/database/user-query"; +import { execute as jupyterExecute } from "@cocalc/server/jupyter/execute"; +import getKernels from "@cocalc/server/jupyter/kernels"; +import getCustomize from "@cocalc/database/settings/customize"; +import callProject from "@cocalc/server/projects/call"; +import isCollaborator from "@cocalc/server/projects/is-collaborator"; + +async function getResponse(endpoint, account_id, params) { + switch (endpoint) { + case "customize": + return await getCustomize(params.fields); + case "user-query": + return { + query: await userQuery({ + ...params, + account_id, + }), + }; + case "exec": + if ( + !(await isCollaborator({ account_id, project_id: params.project_id })) + ) { + throw Error("user must be a collaborator on the project"); + } + return await callProject({ + account_id, + project_id: params.project_id, + mesg: { + event: "project_exec", + ...params, + }, + }); + case "jupyter/execute": + return { + ...(await jupyterExecute({ ...params, account_id })), + success: true, + }; + case "jupyter/kernels": + return { + ...(await getKernels({ ...params, account_id })), + success: true, + }; + + default: + throw Error(`unknown endpoint '${endpoint}'`); + } +} diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts new file mode 100644 index 0000000000..63ab80a90a --- /dev/null +++ b/src/packages/server/nats/index.ts @@ -0,0 +1,12 @@ +import { connect } from "nats"; +import getLogger from "@cocalc/backend/logger"; +import { initAPI } from "./api"; + +const logger = getLogger("server:nats"); + +export default async function initNatsServer() { + logger.debug("initializing nats cocalc hub server"); + const nc = await connect(); + logger.debug(`connected to ${nc.getServer()}`); + initAPI(nc);; +} diff --git a/src/packages/server/package.json b/src/packages/server/package.json index 81c29eeb6b..096eded5a1 100644 --- a/src/packages/server/package.json +++ b/src/packages/server/package.json @@ -12,6 +12,7 @@ "./compute/maintenance": "./dist/compute/maintenance/index.js", "./database/*": "./dist/database/*.js", "./mentions/*": "./dist/mentions/*.js", + "./nats": "./dist/nats/index.js", "./purchases/*": "./dist/purchases/*.js", "./stripe/*": "./dist/stripe/*.js", "./licenses/purchase": "./dist/licenses/purchase/index.js", @@ -22,10 +23,7 @@ "./settings": "./dist/settings/index.js", "./settings/*": "./dist/settings/*.js" }, - "keywords": [ - "utilities", - "cocalc" - ], + "keywords": ["utilities", "cocalc"], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf node_modules dist", @@ -99,6 +97,7 @@ "markdown-it": "^13.0.1", "ms": "2.1.2", "nanoid": "^3.3.8", + "nats": "^2.29.1", "node-zendesk": "^5.0.13", "nodemailer": "^6.9.14", "openai": "^4.52.1", From 3615482f334e5f84ccd835adc4740c616f0c2e4c Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 20 Jan 2025 03:29:09 +0000 Subject: [PATCH 004/281] nats: add proof of concept involving the project --- src/packages/frontend/client/nats.ts | 26 ++- src/packages/pnpm-lock.yaml | 182 ++++++++++++++------- src/packages/project/exec_shell_code.ts | 43 ++--- src/packages/project/package.json | 6 +- src/packages/project/servers/nats/index.ts | 61 +++++++ src/packages/server/nats/api.ts | 18 +- 6 files changed, 242 insertions(+), 94 deletions(-) create mode 100644 src/packages/project/servers/nats/index.ts diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index a5aabb4367..7859926be2 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -32,15 +32,35 @@ export class NatsClient { return this.sc.decode(resp.data); }; - api = async (endpoint: string, params: object) => { + api = async ({ endpoint, params }: { endpoint: string; params?: object }) => { const c = await this.getConnection(); const resp = await c.request( "api.v2", // obviously passing account_id is temporary -- need to use JWT. this.jc.encode({ endpoint, - __account_id: this.client.account_id, - ...params, + account_id: this.client.account_id, + params, + }), + ); + return this.jc.decode(resp.data); + }; + + project = async ({ + project_id, + endpoint, + params, + }: { + project_id: string; + endpoint: string; + params?: object; + }) => { + const c = await this.getConnection(); + const resp = await c.request( + `projects.${project_id}.api`, + this.jc.encode({ + endpoint, + params, }), ); return this.jc.decode(resp.data); diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index fa3952b6b6..b2ca795d11 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -188,7 +188,7 @@ importers: version: 8.7.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) immutable: specifier: ^4.3.0 version: 4.3.7 @@ -393,7 +393,7 @@ importers: version: 1.11.13 debug: specifier: ^4.3.4 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) direction: specifier: ^1.0.4 version: 1.0.4 @@ -796,7 +796,7 @@ importers: version: 2.8.5 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) escape-html: specifier: ^1.0.3 version: 1.0.3 @@ -838,7 +838,7 @@ importers: version: 2.29.1 next: specifier: 14.2.22 - version: 14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.49.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.83.4) + version: 14.2.22(@babel/core@7.25.8)(@opentelemetry/api@1.9.0)(@playwright/test@1.49.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.83.4) nyc: specifier: ^15.1.0 version: 15.1.0 @@ -974,7 +974,7 @@ importers: version: 8.7.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) enchannel-zmq-backend: specifier: ^9.1.23 version: 9.1.23(rxjs@7.8.1) @@ -1249,7 +1249,7 @@ importers: version: 3.0.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) diskusage: specifier: ^1.1.3 version: 1.2.0 @@ -1286,6 +1286,9 @@ importers: lru-cache: specifier: ^7.18.3 version: 7.18.3 + nats: + specifier: ^2.29.1 + version: 2.29.1 pidusage: specifier: ^1.2.0 version: 1.2.0 @@ -1385,7 +1388,7 @@ importers: version: 0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) '@langchain/community': specifier: ^0.3.24 - version: 0.3.24(@browserbasehq/sdk@2.0.0(encoding@0.1.13))(@browserbasehq/stagehand@1.9.0(@playwright/test@1.49.1)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.3.1)(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@11.8.1)(cheerio@1.0.0)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.1)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.1)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.1)(playwright@1.49.1)(ws@8.18.0) + version: 0.3.24(pdmkh75cpnlov7uoawqk3bjgya) '@langchain/core': specifier: ^0.3.30 version: 0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) @@ -1828,7 +1831,7 @@ importers: version: 3.0.0 debug: specifier: ^4.3.4 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) events: specifier: 3.3.0 version: 3.3.0 @@ -1880,7 +1883,7 @@ importers: version: 1.0.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) primus: specifier: ^8.0.9 version: 8.0.9 @@ -1960,7 +1963,7 @@ importers: version: 3.0.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) lodash: specifier: ^4.17.21 version: 4.17.21 @@ -2006,7 +2009,7 @@ importers: version: 1.11.13 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@9.4.0) + version: 4.3.7(supports-color@8.1.1) decimal.js-light: specifier: ^2.5.1 version: 2.5.1 @@ -7973,10 +7976,6 @@ packages: resolution: {integrity: sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==} engines: {node: '>= 4'} - ignore@7.0.3: - resolution: {integrity: sha512-bAH5jbK/F3T3Jls4I0SO1hmPR0dKU0a7+SY6n1yzRtG54FLO8d6w/nxLFX2Nb7dBu6cCWXPaAME6cYqFUMmuCA==} - engines: {node: '>= 4'} - image-extensions@1.1.0: resolution: {integrity: sha512-P0t7ByhK8Jk9TU05ct/7+f7h8dNuXq5OY4m0IO/T+1aga/qHkpC0Wf472x3FLdq/zFDG17pgapCM3JDTxwZzow==} engines: {node: '>=0.10.0'} @@ -12823,7 +12822,7 @@ snapshots: agentkeepalive: 4.6.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.6.7(encoding@0.1.13) transitivePeerDependencies: - encoding @@ -12878,10 +12877,10 @@ snapshots: '@babel/helpers': 7.25.6 '@babel/parser': 7.25.6 '@babel/template': 7.25.0 - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 '@babel/types': 7.25.6 convert-source-map: 2.0.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -12921,7 +12920,7 @@ snapshots: '@babel/traverse': 7.25.7 '@babel/types': 7.25.8 convert-source-map: 2.0.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -13009,14 +13008,21 @@ snapshots: '@babel/helper-optimise-call-expression': 7.24.7 '@babel/helper-replace-supers': 7.25.0(@babel/core@7.25.2) '@babel/helper-skip-transparent-expression-wrappers': 7.24.7 - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 semver: 6.3.1 transitivePeerDependencies: - supports-color '@babel/helper-member-expression-to-functions@7.24.8': dependencies: - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 + '@babel/types': 7.25.6 + transitivePeerDependencies: + - supports-color + + '@babel/helper-module-imports@7.24.7': + dependencies: + '@babel/traverse': 7.25.6 '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -13056,10 +13062,10 @@ snapshots: '@babel/helper-module-transforms@7.25.2(@babel/core@7.25.2)': dependencies: '@babel/core': 7.25.2 - '@babel/helper-module-imports': 7.24.7(supports-color@9.4.0) - '@babel/helper-simple-access': 7.24.7(supports-color@9.4.0) + '@babel/helper-module-imports': 7.24.7 + '@babel/helper-simple-access': 7.24.7 '@babel/helper-validator-identifier': 7.24.7 - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 transitivePeerDependencies: - supports-color @@ -13094,7 +13100,14 @@ snapshots: '@babel/core': 7.25.2 '@babel/helper-member-expression-to-functions': 7.24.8 '@babel/helper-optimise-call-expression': 7.24.7 - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 + transitivePeerDependencies: + - supports-color + + '@babel/helper-simple-access@7.24.7': + dependencies: + '@babel/traverse': 7.25.6 + '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -13114,7 +13127,7 @@ snapshots: '@babel/helper-skip-transparent-expression-wrappers@7.24.7': dependencies: - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -13349,7 +13362,7 @@ snapshots: '@babel/core': 7.25.2 '@babel/helper-module-transforms': 7.25.2(@babel/core@7.25.2) '@babel/helper-plugin-utils': 7.24.8 - '@babel/helper-simple-access': 7.24.7(supports-color@9.4.0) + '@babel/helper-simple-access': 7.24.7 transitivePeerDependencies: - supports-color @@ -13403,6 +13416,18 @@ snapshots: '@babel/parser': 7.25.9 '@babel/types': 7.25.9 + '@babel/traverse@7.25.6': + dependencies: + '@babel/code-frame': 7.24.7 + '@babel/generator': 7.25.6 + '@babel/parser': 7.25.6 + '@babel/template': 7.25.0 + '@babel/types': 7.25.6 + debug: 4.3.7(supports-color@8.1.1) + globals: 11.12.0 + transitivePeerDependencies: + - supports-color + '@babel/traverse@7.25.6(supports-color@9.4.0)': dependencies: '@babel/code-frame': 7.24.7 @@ -13422,7 +13447,7 @@ snapshots: '@babel/parser': 7.25.8 '@babel/template': 7.25.9 '@babel/types': 7.25.8 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -13475,7 +13500,7 @@ snapshots: agentkeepalive: 4.6.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.6.7(encoding@0.1.13) transitivePeerDependencies: - encoding @@ -13524,7 +13549,7 @@ snapshots: awaiting: 3.0.0 cheerio: 1.0.0-rc.12 csv-parse: 5.5.6 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -13538,7 +13563,7 @@ snapshots: '@cocalc/primus-responder@1.0.5': dependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) node-uuid: 1.4.8 transitivePeerDependencies: - supports-color @@ -13811,7 +13836,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -13973,7 +13998,7 @@ snapshots: '@humanwhocodes/config-array@0.13.0': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -13997,7 +14022,7 @@ snapshots: '@antfu/install-pkg': 0.4.1 '@antfu/utils': 0.7.10 '@iconify/types': 2.0.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) kolorist: 1.8.0 local-pkg: 0.5.0 mlly: 1.7.2 @@ -14201,7 +14226,7 @@ snapshots: istanbul-lib-coverage: 3.2.2 istanbul-lib-instrument: 6.0.3 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 4.0.1(supports-color@9.4.0) + istanbul-lib-source-maps: 4.0.1 istanbul-reports: 3.1.7 jest-message-util: 29.7.0 jest-util: 29.7.0 @@ -14469,8 +14494,8 @@ snapshots: transitivePeerDependencies: - encoding - ? '@langchain/community@0.3.24(@browserbasehq/sdk@2.0.0(encoding@0.1.13))(@browserbasehq/stagehand@1.9.0(@playwright/test@1.49.1)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.3.1)(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@11.8.1)(cheerio@1.0.0)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.1)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.1)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.1)(playwright@1.49.1)(ws@8.18.0)' - : dependencies: + '@langchain/community@0.3.24(pdmkh75cpnlov7uoawqk3bjgya)': + dependencies: '@browserbasehq/stagehand': 1.9.0(@playwright/test@1.49.1)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8) '@ibm-cloud/watsonx-ai': 1.3.1 '@langchain/core': 0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) @@ -14492,11 +14517,10 @@ snapshots: '@google-cloud/storage': 7.13.0(encoding@0.1.13) better-sqlite3: 11.8.1 cheerio: 1.0.0 - d3-dsv: 3.0.1 fast-xml-parser: 4.5.1 google-auth-library: 9.14.1(encoding@0.1.13) googleapis: 137.1.0(encoding@0.1.13) - ignore: 7.0.3 + ignore: 5.3.1 jsonwebtoken: 9.0.2 lodash: 4.17.21 pg: 8.13.1 @@ -14813,7 +14837,7 @@ snapshots: '@types/xml-encryption': 1.2.4 '@types/xml2js': 0.4.14 '@xmldom/xmldom': 0.8.10 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) xml-crypto: 3.2.0 xml-encryption: 3.0.2 xml2js: 0.5.0 @@ -15910,7 +15934,7 @@ snapshots: '@typescript-eslint/type-utils': 6.21.0(eslint@8.57.1)(typescript@5.7.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.1)(typescript@5.7.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) eslint: 8.57.1 graphemer: 1.4.0 ignore: 5.3.1 @@ -15928,7 +15952,7 @@ snapshots: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.7.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) eslint: 8.57.1 optionalDependencies: typescript: 5.7.3 @@ -15944,7 +15968,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.7.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.1)(typescript@5.7.3) - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) eslint: 8.57.1 ts-api-utils: 1.3.0(typescript@5.7.3) optionalDependencies: @@ -15958,7 +15982,7 @@ snapshots: dependencies: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 @@ -16161,13 +16185,13 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color agent-base@7.1.1: dependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -18498,7 +18522,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -18882,7 +18906,7 @@ snapshots: follow-redirects@1.15.6(debug@4.3.7): optionalDependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) follow-redirects@1.15.9(debug@4.4.0): optionalDependencies: @@ -19681,7 +19705,7 @@ snapshots: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -19708,21 +19732,21 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.2: dependencies: agent-base: 7.1.1 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.4: dependencies: agent-base: 7.1.1 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -19778,9 +19802,6 @@ snapshots: ignore@5.3.1: {} - ignore@7.0.3: - optional: true - image-extensions@1.1.0: {} image-size@0.5.5: @@ -20177,6 +20198,14 @@ snapshots: make-dir: 4.0.0 supports-color: 7.2.0 + istanbul-lib-source-maps@4.0.1: + dependencies: + debug: 4.3.7(supports-color@8.1.1) + istanbul-lib-coverage: 3.2.2 + source-map: 0.6.1 + transitivePeerDependencies: + - supports-color + istanbul-lib-source-maps@4.0.1(supports-color@9.4.0): dependencies: debug: 4.3.7(supports-color@9.4.0) @@ -21544,6 +21573,34 @@ snapshots: - '@babel/core' - babel-plugin-macros + next@14.2.22(@babel/core@7.25.8)(@opentelemetry/api@1.9.0)(@playwright/test@1.49.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.83.4): + dependencies: + '@next/env': 14.2.22 + '@swc/helpers': 0.5.5 + busboy: 1.6.0 + caniuse-lite: 1.0.30001680 + graceful-fs: 4.2.11 + postcss: 8.4.31 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + styled-jsx: 5.1.1(@babel/core@7.25.8)(react@18.3.1) + optionalDependencies: + '@next/swc-darwin-arm64': 14.2.22 + '@next/swc-darwin-x64': 14.2.22 + '@next/swc-linux-arm64-gnu': 14.2.22 + '@next/swc-linux-arm64-musl': 14.2.22 + '@next/swc-linux-x64-gnu': 14.2.22 + '@next/swc-linux-x64-musl': 14.2.22 + '@next/swc-win32-arm64-msvc': 14.2.22 + '@next/swc-win32-ia32-msvc': 14.2.22 + '@next/swc-win32-x64-msvc': 14.2.22 + '@opentelemetry/api': 1.9.0 + '@playwright/test': 1.49.1 + sass: 1.83.4 + transitivePeerDependencies: + - '@babel/core' + - babel-plugin-macros + nise@1.5.3: dependencies: '@sinonjs/formatio': 3.2.2 @@ -21720,7 +21777,7 @@ snapshots: istanbul-lib-instrument: 4.0.3 istanbul-lib-processinfo: 2.0.3 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 4.0.1(supports-color@9.4.0) + istanbul-lib-source-maps: 4.0.1 istanbul-reports: 3.1.7 make-dir: 3.1.0 node-preload: 0.2.1 @@ -23995,7 +24052,7 @@ snapshots: spdy-transport@3.0.0: dependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) detect-node: 2.1.0 hpack.js: 2.1.6 obuf: 1.1.2 @@ -24006,7 +24063,7 @@ snapshots: spdy@4.0.2: dependencies: - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) handle-thing: 2.0.1 http-deceiver: 1.2.7 select-hose: 2.0.0 @@ -24216,6 +24273,13 @@ snapshots: optionalDependencies: '@babel/core': 7.25.2 + styled-jsx@5.1.1(@babel/core@7.25.8)(react@18.3.1): + dependencies: + client-only: 0.0.1 + react: 18.3.1 + optionalDependencies: + '@babel/core': 7.25.8 + stylis@4.3.4: {} superb@3.0.0: @@ -25133,7 +25197,7 @@ snapshots: dependencies: '@wwa/statvfs': 1.1.18 awaiting: 3.0.0 - debug: 4.3.7(supports-color@9.4.0) + debug: 4.3.7(supports-color@8.1.1) port-get: 1.0.4 ws: 8.18.0 transitivePeerDependencies: diff --git a/src/packages/project/exec_shell_code.ts b/src/packages/project/exec_shell_code.ts index e8a8f85977..02b0f76609 100644 --- a/src/packages/project/exec_shell_code.ts +++ b/src/packages/project/exec_shell_code.ts @@ -28,26 +28,7 @@ export async function exec_shell_code(socket: CoCalcSocket, mesg) { D(`command=${mesg.command} args=${mesg.args} path=${mesg.path}`); try { - const out = await execCode({ - path: !!mesg.compute_server_id - ? mesg.path - : abspath(mesg.path != null ? mesg.path : ""), - ...mesg, - }); - let ret: ExecuteCodeOutput & { id: string } = { - id: mesg.id, - type: "blocking", - stdout: out?.stdout, - stderr: out?.stderr, - exit_code: out?.exit_code, - }; - if (out?.type === "async") { - // extra fields for ExecuteCodeOutputAsync - ret = { - ...ret, - ...out, // type=async, pid, status, job_id, stats, ... - }; - } + const ret = handleExecShellCode(mesg); socket.write_mesg("json", message.project_exec_output(ret)); } catch (err) { let error = `Error executing command '${mesg.command}' with args '${mesg.args}' -- ${err}`; @@ -65,3 +46,25 @@ export async function exec_shell_code(socket: CoCalcSocket, mesg) { socket.write_mesg("json", err_mesg); } } + +export async function handleExecShellCode(mesg) { + const out = await execCode({ + path: !!mesg.compute_server_id ? mesg.path : abspath(mesg.path ?? ""), + ...mesg, + }); + let ret: ExecuteCodeOutput & { id: string } = { + id: mesg.id, + type: "blocking", + stdout: out?.stdout, + stderr: out?.stderr, + exit_code: out?.exit_code, + }; + if (out?.type === "async") { + // extra fields for ExecuteCodeOutputAsync + ret = { + ...ret, + ...out, // type=async, pid, status, job_id, stats, ... + }; + } + return ret; +} diff --git a/src/packages/project/package.json b/src/packages/project/package.json index 58cd32d11f..38c23659bb 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -4,6 +4,7 @@ "description": "CoCalc: project daemon", "exports": { "./named-servers": "./dist/named-servers/index.js", + "./servers/nats": "./dist/servers/nats/index.js", "./*": "./dist/*.js" }, "keywords": [ @@ -52,6 +53,7 @@ "lean-client-js-node": "^1.2.12", "lodash": "^4.17.21", "lru-cache": "^7.18.3", + "nats": "^2.29.1", "pidusage": "^1.2.0", "prettier": "^3.0.2", "primus": "^8.0.9", @@ -82,9 +84,7 @@ "clean": "rm -rf dist" }, "author": "SageMath, Inc.", - "contributors": [ - "William Stein " - ], + "contributors": ["William Stein "], "license": "SEE LICENSE.md", "bugs": { "url": "https://github.com/sagemathinc/cocalc/issues" diff --git a/src/packages/project/servers/nats/index.ts b/src/packages/project/servers/nats/index.ts new file mode 100644 index 0000000000..63c5a733e1 --- /dev/null +++ b/src/packages/project/servers/nats/index.ts @@ -0,0 +1,61 @@ +/* +Run for a project_id you want to simulate: + + export HOME=... # (optional) + export COCALC_PROJECT_ID='81e0c408-ac65-4114-bad5-5f4b6539bd0e' + echo 'require("@cocalc/project/servers/nats").default()' | node + +then in the browser: + + await cc.client.nats_client.project({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e', endpoint:"exec", params:{command:'echo $COCALC_PROJECT_ID'}}) + +*/ + +import { getLogger } from "@cocalc/project/logger"; +import { connect, JSONCodec } from "nats"; +import { project_id } from "@cocalc/project/data"; + +const logger = getLogger("server:nats"); + +export default async function initNatsServer() { + logger.debug("initializing nats cocalc project server"); + const nc = await connect(); + logger.debug(`connected to ${nc.getServer()}`); + initAPI(nc); +} + +const jc = JSONCodec(); + +export async function initAPI(nc) { + const subject = `projects.${project_id}.api`; + logger.debug(`initAPI -- NATS project subject '${subject}'`); + const sub = nc.subscribe(subject); + for await (const mesg of sub) { + handleApiRequest(mesg); + } +} + +async function handleApiRequest(mesg) { + const request = jc.decode(mesg.data) ?? {}; + let resp; + try { + // TODO: obviously user-provided account_id is no good! This is a POC. + const { endpoint, params } = request as any; + logger.debug("handling project request:", { endpoint }); + resp = await getResponse({ endpoint, params }); + } catch (err) { + resp = { error: `${err}` }; + } + mesg.respond(jc.encode(resp)); +} + +import { handleExecShellCode } from "@cocalc/project/exec_shell_code"; + +async function getResponse({ endpoint, params }) { + switch (endpoint) { + case "exec": + return await handleExecShellCode(params); + default: + throw Error(`unknown endpoint '${endpoint}'`); + } +} diff --git a/src/packages/server/nats/api.ts b/src/packages/server/nats/api.ts index 57ccdaeca0..e0e6c5e8b0 100644 --- a/src/packages/server/nats/api.ts +++ b/src/packages/server/nats/api.ts @@ -1,19 +1,19 @@ /* This is meant to be similar to the nexts pages http api/v2, but using NATS instead of HTTPS. -To do development turn off for the hub, and run like this: +To do development turn off nats-server handling for the hub, and run this script standalone: echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node + +To make use of this from a browser: -When you make changes, just restart it the above. All clients will instantly + await cc.client.nats_client.api({endpoint:"customize", params:{fields:['siteName']}}) + +When you make changes, just restart the above. All clients will instantly use the new version after you restart, and there is no need to restart the hub itself or any clients. -To view all requests in realtime: - - nats sub api.v2 - -and if you want to also see the replies: +To view all requests (and replies) in realtime: nats sub api.v2 --match-replies */ @@ -38,7 +38,7 @@ async function handleApiRequest(mesg) { let resp; try { // TODO: obviously user-provided account_id is no good! This is a POC. - const { endpoint, __account_id: account_id, ...params } = request as any; + const { endpoint, account_id, params } = request as any; logger.debug("handling api.v2 request:", { endpoint }); resp = await getResponse(endpoint, account_id, params); } catch (err) { @@ -57,7 +57,7 @@ import isCollaborator from "@cocalc/server/projects/is-collaborator"; async function getResponse(endpoint, account_id, params) { switch (endpoint) { case "customize": - return await getCustomize(params.fields); + return await getCustomize(params?.fields); case "user-query": return { query: await userQuery({ From f5ed6b8e7ab5981000e96016045257c8c0b1830a Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 20 Jan 2025 03:45:41 +0000 Subject: [PATCH 005/281] nats: use queue group for hub.api POC server --- src/packages/frontend/client/nats.ts | 2 +- src/packages/server/nats/api.ts | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 7859926be2..c04db68453 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -35,7 +35,7 @@ export class NatsClient { api = async ({ endpoint, params }: { endpoint: string; params?: object }) => { const c = await this.getConnection(); const resp = await c.request( - "api.v2", + "hub.api", // obviously passing account_id is temporary -- need to use JWT. this.jc.encode({ endpoint, diff --git a/src/packages/server/nats/api.ts b/src/packages/server/nats/api.ts index e0e6c5e8b0..760e22a540 100644 --- a/src/packages/server/nats/api.ts +++ b/src/packages/server/nats/api.ts @@ -5,6 +5,12 @@ To do development turn off nats-server handling for the hub, and run this script echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node +Optional: start more servers -- requests get randomly routed to exactly one of them: + + echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node + echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node + + To make use of this from a browser: await cc.client.nats_client.api({endpoint:"customize", params:{fields:['siteName']}}) @@ -26,8 +32,8 @@ const logger = getLogger("server:nats"); const jc = JSONCodec(); export async function initAPI(nc) { - logger.debug("initAPI -- NATS api.v2 subject"); - const sub = nc.subscribe("api.v2"); + logger.debug("initAPI -- subject='hub.api', options=", { queue: "0" }); + const sub = nc.subscribe("hub.api", { queue: "0" }); for await (const mesg of sub) { handleApiRequest(mesg); } @@ -39,7 +45,7 @@ async function handleApiRequest(mesg) { try { // TODO: obviously user-provided account_id is no good! This is a POC. const { endpoint, account_id, params } = request as any; - logger.debug("handling api.v2 request:", { endpoint }); + logger.debug("handling hub.api request:", { endpoint }); resp = await getResponse(endpoint, account_id, params); } catch (err) { resp = { error: `${err}` }; From 5e84190be0bf7befff56353d78b9e25f6907b93e Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 20 Jan 2025 21:35:57 +0000 Subject: [PATCH 006/281] nats POC: switching to using "ability to publish to the subject" to prove identity... --- src/packages/frontend/client/nats.ts | 9 ++++++--- src/packages/server/nats/api.ts | 15 +++++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index c04db68453..85a27aa337 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -34,16 +34,19 @@ export class NatsClient { api = async ({ endpoint, params }: { endpoint: string; params?: object }) => { const c = await this.getConnection(); + const subject = `hub.api.${this.client.account_id}`; + console.log(`publishing to subject='${subject}'`); const resp = await c.request( - "hub.api", - // obviously passing account_id is temporary -- need to use JWT. + subject, this.jc.encode({ endpoint, account_id: this.client.account_id, params, }), ); - return this.jc.decode(resp.data); + const x = this.jc.decode(resp.data); + console.log("got back ", x); + return x; }; project = async ({ diff --git a/src/packages/server/nats/api.ts b/src/packages/server/nats/api.ts index 760e22a540..c55db0fdb9 100644 --- a/src/packages/server/nats/api.ts +++ b/src/packages/server/nats/api.ts @@ -26,6 +26,7 @@ To view all requests (and replies) in realtime: import { JSONCodec } from "nats"; import getLogger from "@cocalc/backend/logger"; +import { isValidUUID } from "@cocalc/util/misc"; const logger = getLogger("server:nats"); @@ -33,19 +34,25 @@ const jc = JSONCodec(); export async function initAPI(nc) { logger.debug("initAPI -- subject='hub.api', options=", { queue: "0" }); - const sub = nc.subscribe("hub.api", { queue: "0" }); + const sub = nc.subscribe("hub.api.>", { queue: "0" }); for await (const mesg of sub) { handleApiRequest(mesg); } } async function handleApiRequest(mesg) { - const request = jc.decode(mesg.data) ?? {}; + console.log({ subject: mesg.subject }); let resp; try { + const segments = mesg.subject.split("."); + const account_id = segments[2]; + if (!isValidUUID(account_id)) { + throw Error(`invalid account_id '${account_id}'`); + } + const request = jc.decode(mesg.data) ?? {}; // TODO: obviously user-provided account_id is no good! This is a POC. - const { endpoint, account_id, params } = request as any; - logger.debug("handling hub.api request:", { endpoint }); + const { endpoint, params } = request as any; + logger.debug("handling hub.api request:", { account_id, endpoint, params }); resp = await getResponse(endpoint, account_id, params); } catch (err) { resp = { error: `${err}` }; From 52a18ad33081ffd8851e13815c07acf596c96620 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 20 Jan 2025 22:17:56 +0000 Subject: [PATCH 007/281] NATS POC - work in progress exploring how to do auth --- src/packages/frontend/client/nats.ts | 11 ++++++++++- src/packages/server/nats/index.ts | 23 ++++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 85a27aa337..c88fc4d1d0 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -10,18 +10,27 @@ export class NatsClient { private nc?: Awaited>; // obviously just for learning: public nats = nats; + private creds = ""; constructor(client: WebappClient) { this.client = client; } getConnection = reuseInFlight(async () => { + if (!this.creds) { + throw Error("set this.creds first"); + } if (this.nc != null) { return this.nc; } const server = `${location.protocol == "https:" ? "wss" : "ws"}://${location.host}${appBasePath}/nats`; console.log(`connecting to ${server}...`); - this.nc = await nats.connect({ servers: [server] }); + this.nc = await nats.connect({ + servers: [server], + authenticator: nats.credsAuthenticator( + new TextEncoder().encode(this.creds), + ), + }); console.log(`connected to ${server}`); return this.nc; }); diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts index 63ab80a90a..e3291c38e1 100644 --- a/src/packages/server/nats/index.ts +++ b/src/packages/server/nats/index.ts @@ -1,12 +1,29 @@ -import { connect } from "nats"; +import { connect, credsAuthenticator } from "nats"; import getLogger from "@cocalc/backend/logger"; import { initAPI } from "./api"; const logger = getLogger("server:nats"); +const creds = `-----BEGIN NATS USER JWT----- +eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJNSEtXWTU0WDVNSUxXUEZBRkdRTFZVRkdTT0VCSDZUMjMyVjUzRzVRSjI3RFJWTFhOUk1BIiwiaWF0IjoxNzM3NDEwMTM4LCJpc3MiOiJBQjJLVEVGUFIyTzc2UE9aVVBZRVFTS1RaQVg2R0lOQVZUNkpXU0g2UUI3TENNNFhIRlRITVgyTCIsIm5hbWUiOiJodWIiLCJzdWIiOiJVQUhTQ1hVUVEzSFVVRlFIV0xUN0tYVDRXSU1ZSkdSQ1VLUUROWEZQRURCNU1WWkNMNkJKTldWVSIsIm5hdHMiOnsicHViIjp7ImFsbG93IjpbIl9JTkJPWC5cdTAwM2UiXX0sInN1YiI6eyJhbGxvdyI6WyJodWIuYXBpLlx1MDAzZSJdfSwic3VicyI6LTEsImRhdGEiOi0xLCJwYXlsb2FkIjotMSwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.3lQtNIOlj1nEStkbTSz4j25T1tIYfshWVvjyYE-ArkcmmCpzynoYCxEyKF8QuMNdZ30YTYu6xxcqluzjVCk_DA +------END NATS USER JWT------ + +************************* IMPORTANT ************************* +NKEY Seed printed below can be used to sign and prove identity. +NKEYs are sensitive and should be treated as secrets. + +-----BEGIN USER NKEY SEED----- +SUAOMBSB4Z6XVTXWQCVZPG2OWM6C36UTP6O47ILTW3LC75HW5U2QCE3C5U +------END USER NKEY SEED------ + +************************************************************* +`; + export default async function initNatsServer() { logger.debug("initializing nats cocalc hub server"); - const nc = await connect(); + const nc = await connect({ + authenticator: credsAuthenticator(new TextEncoder().encode(creds)), + }); logger.debug(`connected to ${nc.getServer()}`); - initAPI(nc);; + initAPI(nc); } From 9cd0cd542141e533b0279027a987de65b1673216 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 20 Jan 2025 23:36:00 +0000 Subject: [PATCH 008/281] NATS POC -- figured out *how* to securely do auth with a JWT from a browser client - the key idea is a "bearer JWT". The bearer part took a while to understand --- src/packages/frontend/client/nats.ts | 7 - src/packages/hub/servers/nats.ts | 1 + .../next/pages/api/v2/auth/sign-in.ts | 5 + src/packages/server/nats/notes.md | 123 ++++++++++++++++++ 4 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 src/packages/server/nats/notes.md diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index c88fc4d1d0..b44c98613f 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -10,16 +10,12 @@ export class NatsClient { private nc?: Awaited>; // obviously just for learning: public nats = nats; - private creds = ""; constructor(client: WebappClient) { this.client = client; } getConnection = reuseInFlight(async () => { - if (!this.creds) { - throw Error("set this.creds first"); - } if (this.nc != null) { return this.nc; } @@ -27,9 +23,6 @@ export class NatsClient { console.log(`connecting to ${server}...`); this.nc = await nats.connect({ servers: [server], - authenticator: nats.credsAuthenticator( - new TextEncoder().encode(this.creds), - ), }); console.log(`connected to ${server}`); return this.nc; diff --git a/src/packages/hub/servers/nats.ts b/src/packages/hub/servers/nats.ts index aaa35f61b5..80a087dfea 100644 --- a/src/packages/hub/servers/nats.ts +++ b/src/packages/hub/servers/nats.ts @@ -7,6 +7,7 @@ We assume there is a NATS server running on localhost with this configuration: websocket { listen: "localhost:8443" no_tls: true + jwt_cookie: "cocalc_nats_jwt_cookie" } You could start this with diff --git a/src/packages/next/pages/api/v2/auth/sign-in.ts b/src/packages/next/pages/api/v2/auth/sign-in.ts index 8f6f81eca0..42fadfa1fe 100644 --- a/src/packages/next/pages/api/v2/auth/sign-in.ts +++ b/src/packages/next/pages/api/v2/auth/sign-in.ts @@ -91,6 +91,11 @@ export async function signUserIn(req, res, account_id: string): Promise { maxAge: ttl_s * 1000, sameSite: samesite_remember_me, }); + // todo: NATS POC! + cookies.set( + "cocalc_nats_jwt_cookie", + "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiI1QjNKWVozNjJTUDdHUzZGU05JNzJRRFkyQVM1RUVYSUNERFNIM01GSkk3R0k2TTdRUUZBIiwiaWF0IjoxNzM3NDE1NTcxLCJpc3MiOiJBQjJLVEVGUFIyTzc2UE9aVVBZRVFTS1RaQVg2R0lOQVZUNkpXU0g2UUI3TENNNFhIRlRITVgyTCIsIm5hbWUiOiJ3c3RlaW4iLCJzdWIiOiJVQVFXUFc3QktRSkJLWUwzQUtUMjdIRE43UFpONEZJSlNUUDRCU0o1MktOUlhaM1ZLWk9QNk1DRiIsIm5hdHMiOnsicHViIjp7ImFsbG93IjpbIl9JTkJPWC5cdTAwM2UiLCJodWIuYXBpLjI3NWYxZGI3LWJmMzctNGI0NC1iOWFhLWQ2NDY5NDI2OWM5ZiJdfSwic3ViIjp7ImFsbG93IjpbIl9JTkJPWC5cdTAwM2UiLCJodWIuYXBpLjI3NWYxZGI3LWJmMzctNGI0NC1iOWFhLWQ2NDY5NDI2OWM5ZiJdfSwic3VicyI6LTEsImRhdGEiOi0xLCJwYXlsb2FkIjotMSwiYmVhcmVyX3Rva2VuIjp0cnVlLCJ0eXBlIjoidXNlciIsInZlcnNpb24iOjJ9fQ.JQBl6OoCCFmuxgoIgA7lrE_75Ut_MQ6hfu9W9Y7BW2Wvz-Bq7kxCZ_kKWhoLR8sH4RBQokwv48alGkXKzbnsAQ", + ); } catch (err) { res.json({ error: `Problem setting cookie -- ${err.message}.` }); return; diff --git a/src/packages/server/nats/notes.md b/src/packages/server/nats/notes.md new file mode 100644 index 0000000000..065874064b --- /dev/null +++ b/src/packages/server/nats/notes.md @@ -0,0 +1,123 @@ +# Systematically keep track of NATS experiment here! + +## [x] Goal: nats from nodejs + +- start a nats server in cocalc\-docker +- connect from nats cli outside docker +- connect to it from the nodejs client over a websocket + +```sh +nats-server -p 5004 + +nats context save --select --server nats://localhost:5004 nats + +nats sub '>' +``` + +Millions of messages a second works \-\- and you can run like 5x of these at once without saturating nats\-server. + +```js +import { connect, StringCodec } from "nats"; +const nc = await connect({ port: 5004 }); +console.log(`connected to ${nc.getServer()}`); +const sc = StringCodec(); + +const t0 = Date.now(); +for (let i = 0; i < 1000000; i++) { + nc.publish("hello", sc.encode("world")); +} +await nc.drain(); +console.log(Date.now() - t0); +``` + +That was connecting over TCP. Now can we connect via websocket? + +## [x] Goal: Websocket from browser + +First need to start a nats **websocket** server instead on port 5004: + +[https://nats.io/blog/getting\-started\-nats\-ws/](https://nats.io/blog/getting-started-nats-ws/) + +```sh +nats context save --select --server ws://localhost:5004 ws +~/nats/nats.js/lib$ nats context select ws +NATS Configuration Context "ws" + + Server URLs: ws://localhost:5004 + Path: /projects/3fa218e5-7196-4020-8b30-e2127847cc4f/.config/nats/context/ws.json + +~/nats/nats.js/lib$ nats pub foo bar +21:24:53 Published 3 bytes to "foo" +~/nats/nats.js/lib$ +``` + +## + +- their no\-framework html example DOES work for me! +- [https://localhost:4043/projects/3fa218e5\-7196\-4020\-8b30\-e2127847cc4f/files/nats/nats.js/lib/ws.html](https://localhost:4043/projects/3fa218e5-7196-4020-8b30-e2127847cc4f/files/nats/nats.js/lib/ws.html) +- It takes about 1\-2 seconds to send **one million messages** from browser outside docker to what is running inside there! + +## [x] Goal: actually do something useful + +- nats server +- browser connects via websocket port 5004 +- nodejs hub connects via tcp +- hub answers a ping or something else from the browser... + +This worked perfectly with no difficulty. It's very fast and flexible and robust. + +Reconnects work, etc. + +## [x] Goal: proxying + +- nats server with websocket listening on localhost:5004 +- proxy it via node\-proxy in the hub to localhost:4043/nats +- as above + +This totally worked! + +Everything is working that I try?! + +Maybe NATS totally kicks ass. + +## [x] Goal: do something actually useful. + +- authentication: is there a way to too who the user who made the websocket connection is? + - worry about this **later** \- obviously possible and not needed for a POC +- let's try to make `write_text_file_to_project` also be possible via nats. +- OK, made some of api/v2 usable. Obviously this is really minimal POC. + +## [x] GOAL: do something involving the project + +The most interesting use case for nats/jetsteam is timetravel collab editing, where this is all a VERY natural fit. + +But for now, let's just do *something* at all. + +This worked - I did project exec with subject projects.{project_id}.api + +## [x] Goal: Queue group for hub api + +- change this to be a queue group and test by starting a few servers at once + +## [x] Goal: Auth Strategy that is meaningful + +Creating a creds file that encodes a JWT that says what you can publish and subscribe to, then authenticating with that works. + +- make it so user with account\_id can publish to hub.api.{account\_id} makes it so we know the account\_id automatically by virtue of what was published to. This works. + +## [ ] Goal: Solve Critical Auth Problems + +Now need to solve two problems: + +- [x] GOAL: set the creds for a browser client in a secure http cookie, so the browser can't directly access it + +I finally figured this out after WASTING a lot of time with stupid AI misleading me and trying actively to get me to write very stupid insecure code as a lazy workaround. AI really is very, very dangerous... The trick was to read the docs repeatedly, increase logging a lot, and \-\- most imporantly \-\- read the relevant Go source code of NATS itself. The answer is to modify the JWT so that it explicitly has bearer set: `nsc edit user wstein --bearer` + +This makes it so the server doesn't check the signature of the JWT against the _user_ . Putting exactly the JWT token string in the cookie then works because "bearer" literally tells the backend server not to do the signature check. I think this is secure and the right approach because the server checks that the JWT is valid using the account and operator signatures. + +- [ ] GOAL: automate creation of creds for browser clients, i.e., what we just did with the nsc tool. + +If we can do the above, we should also be able to do something similar for: + +- [ ] GOAL: connecting to projects. I.e., access to a project means you can publish to `projects.{project_id}.>` Also, projects should have access to something under hub. + From 7409e81c06f7fbc339902fc2a3600ce67bf95741 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 21 Jan 2025 19:19:24 +0000 Subject: [PATCH 009/281] nats: implement creating nats user associated to account, including access to project subjects --- src/packages/server/nats/api.ts | 2 +- src/packages/server/nats/auth.ts | 233 +++++++++++++++++++++++++++++++ 2 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 src/packages/server/nats/auth.ts diff --git a/src/packages/server/nats/api.ts b/src/packages/server/nats/api.ts index c55db0fdb9..a5af8ce653 100644 --- a/src/packages/server/nats/api.ts +++ b/src/packages/server/nats/api.ts @@ -28,7 +28,7 @@ import { JSONCodec } from "nats"; import getLogger from "@cocalc/backend/logger"; import { isValidUUID } from "@cocalc/util/misc"; -const logger = getLogger("server:nats"); +const logger = getLogger("server:nats:api"); const jc = JSONCodec(); diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts new file mode 100644 index 0000000000..702b6373ba --- /dev/null +++ b/src/packages/server/nats/auth.ts @@ -0,0 +1,233 @@ +/* +Points that took me a while to figure out: + +- For each CoCalc user who is accessing CoCalc resources from a *browser*, on the fly, we create: + + - a signing key, which allows access to hub api's and running projects they are a collaborator on, etc., etc. + + - a NATS user that has *bearer* enabled and is associated to the above signing key, so they get all its permissions + + Then the JWT for the user is stored as a secure http cookie in the browser and grants the user permissions. + TODO: worry about expiration + +- There is no supported way to do user management except calling the nsc command line tool. That's fine. +*/ + +import { executeCode } from "@cocalc/backend/execute-code"; +import getPool from "@cocalc/database/pool"; +import { isValidUUID } from "@cocalc/util/misc"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("server:nats:auth"); + +export async function nsc(args: string[]) { + // todo: for production we have to put some authentication + // options, e.g., taken from the database. Skip that for now. + console.log(`nsc ${args.join(" ")}`); + return await executeCode({ command: "nsc", args }); +} + +// TODO: consider making the names shorter strings using https://www.npmjs.com/package/short-uuid + +// A CoCalc User is (so far): a project or an account. +type CoCalcUser = + | { + account_id: string; + project_id?: string; + } + | { + account_id?: string; + project_id: string; + }; + +function getCoCalcUserType({ + account_id, + project_id, +}: CoCalcUser): "account" | "project" { + if (account_id) { + if (project_id) { + throw Error("exactly one of account_id or project_id must be specified"); + } + return "account"; + } + if (project_id) { + return "project"; + } + throw Error("account_id or project_id must be specified"); +} + +function getCoCalcUserId({ account_id, project_id }: CoCalcUser): string { + if (account_id) { + if (project_id) { + throw Error("exactly one of account_id or project_id must be specified"); + } + return account_id; + } + if (project_id) { + return project_id; + } + throw Error("account_id or project_id must be specified"); +} + +function getNatsUserName({ account_id, project_id }: CoCalcUser) { + if (account_id) { + if (project_id) { + throw Error("exactly one of account_id or project_id must be specified"); + } + return `account-${account_id}`; + } + if (project_id) { + return `project-${project_id}`; + } + throw Error("account_id or project_id must be specified"); +} + +export async function getNatsUserJwt(cocalcUser: CoCalcUser): Promise { + return ( + await nsc(["describe", "user", getNatsUserName(cocalcUser), "--raw"]) + ).stdout.trim(); +} + +export async function configureNatsUser(cocalcUser: CoCalcUser) { + const name = getNatsUserName(cocalcUser); + const userId = getCoCalcUserId(cocalcUser); + if (!isValidUUID(userId)) { + throw Error("must be a valid uuid"); + } + const userType = getCoCalcUserType(cocalcUser); + const goalPub = new Set([`hub.${userType}.${userId}.>`]); + const goalSub = new Set(["_INBOX.>"]); + + if (userType == "account") { + const pool = getPool(); + // all RUNNING projects with the user's group + const query = `SELECT project_id, users#>>'{${userId},group}' AS group FROM projects WHERE state#>>'{state}'='running' AND users ? '${userId}' ORDER BY project_id`; + const { rows } = await pool.query(query); + for (const { project_id, group } of rows) { + goalPub.add(`project.${project_id}.${group}.${userId}.>`); + } + // TODO: there will be other subjects + // TODO: something similar for projects, e.g., they can publish to a channel that browser clients + // will listen to, e.g., for timetravel editing. + } + + // **Subject Permissions SYNC Algorithm ** + // figure out what signing key currently allows an update it to be exactly what is specified above. + + const currentSigningKey = await getScopedSigningKey(name); + if (currentSigningKey == null) { + throw Error(`no signing key '${name}'`); + } + const currentPub = new Set(currentSigningKey["Pub Allow"] ?? []); + const currentSub = new Set(currentSigningKey["Sub Allow"] ?? []); + const rm: string[] = []; + const pub: string[] = []; + const sub: string[] = []; + for (const subject of goalPub) { + if (!currentPub.has(subject)) { + // need to add: + pub.push(subject); + } + } + for (const subject of currentPub) { + if (!goalPub.has(subject)) { + // need to remove + rm.push(subject); + // this removes from everything after it happens, so update currenSub state, just in case + currentSub.delete(subject); + } + } + for (const subject of goalSub) { + if (!currentSub.has(subject)) { + // need to add: + sub.push(subject); + } + } + for (const subject of currentSub) { + if (!goalSub.has(subject)) { + // need to remove + rm.push(subject); + // does this break a pub? if so, account for this. + if (goalPub.has(subject) && !pub.includes(subject)) { + pub.push(subject); + } + } + } + const args = ["edit", "signing-key", "--sk", name]; + if (rm.length > 0) { + // --rm applies to both pub and sub and happens after adding, + // so we have to do it separately at the beginning in order to + // handle some edge cases (that might never happen). + logger.debug("configureNatsUser ", { rm }); + await nsc([...args, "--rm", rm.join(",")]); + } + if (sub.length > 0 || pub.length > 0) { + if (sub.length > 0) { + args.push("--allow-sub"); + args.push(sub.join(",")); + } + if (pub.length > 0) { + args.push("--allow-pub"); + args.push(pub.join(",")); + } + logger.debug("configureNatsUser ", { pub, sub }); + await nsc(args); + } +} + +export async function getScopedSigningKey(natsUser: string) { + let { stdout } = await nsc(["describe", "user", natsUser]); + // it seems impossible to get the scoped signing key params using --json; they just aren't there + // i.e., it's not implemented. so we parse it. + const i = stdout.indexOf("Scoped"); + if (i == -1) { + // there is no scoped signing key + return null; + } + stdout = stdout.slice(i); + const obj: any = {}; + let key = ""; + for (const line of stdout.split("\n")) { + const v = line.split("|"); + if (v.length == 4) { + const key2 = v[1].trim(); + let val: string | string[] = v[2].trim(); + if (!key2 && obj[key] != null) { + // automatically account for arrays (for pub and sub) + if (typeof obj[key] == "string") { + obj[key] = [obj[key], val]; + } else { + obj[key].push(val); + } + } else { + key = key2; + if (key.startsWith("Pub ") || key.startsWith("Sub ")) { + val = [val]; + } + obj[key] = val; + } + } + } + return obj; +} + +export async function createNatsUser(cocalcUser: CoCalcUser) { + const { stderr } = await nsc(["edit", "account", "--sk", "generate"]); + const key = stderr.trim().split('"')[1]; + const name = getNatsUserName(cocalcUser); + // bearer is critical so that the signing key can be used in the browser without + // requiring the private key to also be in the client in the browser (which is + // less secure since it easily leaks). + await nsc(["edit", "signing-key", "--sk", key, "--role", name, "--bearer"]); + await nsc(["add", "user", name, "--private-key", name]); + await configureNatsUser(cocalcUser); +} + +export async function getJwt(cocalcUser: CoCalcUser): Promise { + try { + return await getNatsUserJwt(cocalcUser); + } catch (_err) { + await createNatsUser(cocalcUser); + return await getNatsUserJwt(cocalcUser); + } +} From 737fb73f8890429cc0d3e80c3a640c0b49d76e2e Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 21 Jan 2025 21:10:26 +0000 Subject: [PATCH 010/281] nats: setting nats jwt cookie automatically and in the right place --- src/packages/backend/auth/cookie-names.ts | 2 ++ src/packages/frontend/client/nats.ts | 17 ++++++--- src/packages/hub/servers/express-app.ts | 2 ++ src/packages/hub/servers/nats.ts | 36 +++++++++++++++++++ .../next/pages/api/v2/auth/sign-in.ts | 5 --- src/packages/server/nats/api.ts | 8 +++-- src/packages/server/nats/auth.ts | 4 +-- src/packages/server/nats/index.ts | 2 +- 8 files changed, 61 insertions(+), 15 deletions(-) diff --git a/src/packages/backend/auth/cookie-names.ts b/src/packages/backend/auth/cookie-names.ts index 8569ad4e04..7468aa6f29 100644 --- a/src/packages/backend/auth/cookie-names.ts +++ b/src/packages/backend/auth/cookie-names.ts @@ -32,3 +32,5 @@ export const API_COOKIE_NAME = `${basePath.length <= 1 ? "" : encodeURIComponent(basePath)}api_key`; log.debug("API_COOKIE_NAME", API_COOKIE_NAME); + +export const NATS_JWT_COOKIE_NAME = `${basePath.length <= 1 ? "" : encodeURIComponent(basePath)}cocalc_nats_jwt_cookie`; diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index b44c98613f..ed3998fc1d 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -2,6 +2,7 @@ import * as nats from "nats.ws"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import type { WebappClient } from "./client"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { join } from "path"; export class NatsClient { /*private*/ client: WebappClient; @@ -21,9 +22,17 @@ export class NatsClient { } const server = `${location.protocol == "https:" ? "wss" : "ws"}://${location.host}${appBasePath}/nats`; console.log(`connecting to ${server}...`); - this.nc = await nats.connect({ - servers: [server], - }); + try { + this.nc = await nats.connect({ + servers: [server], + }); + } catch (err) { + console.log("set the JWT cookie and try again"); + await fetch(join(appBasePath, "nats")); + this.nc = await nats.connect({ + servers: [server], + }); + } console.log(`connected to ${server}`); return this.nc; }); @@ -36,7 +45,7 @@ export class NatsClient { api = async ({ endpoint, params }: { endpoint: string; params?: object }) => { const c = await this.getConnection(); - const subject = `hub.api.${this.client.account_id}`; + const subject = `hub.account.api.${this.client.account_id}`; console.log(`publishing to subject='${subject}'`); const resp = await c.request( subject, diff --git a/src/packages/hub/servers/express-app.ts b/src/packages/hub/servers/express-app.ts index 55f9e5dea0..c632987730 100644 --- a/src/packages/hub/servers/express-app.ts +++ b/src/packages/hub/servers/express-app.ts @@ -31,6 +31,7 @@ import initStats from "./app/stats"; import { database } from "./database"; import initHttpServer from "./http"; import initRobots from "./robots"; +import { initNatsServer } from "./nats"; // Used for longterm caching of files. This should be in units of seconds. const MAX_AGE = Math.round(ms("10 days") / 1000); @@ -126,6 +127,7 @@ export default async function init(opts: Options): Promise<{ initBlobs(router); initBlobUpload(router); initSetCookies(router); + initNatsServer(router); initCustomize(router, opts.isPersonal); initStats(router); initAppRedirect(router); diff --git a/src/packages/hub/servers/nats.ts b/src/packages/hub/servers/nats.ts index 80a087dfea..11d67d0f99 100644 --- a/src/packages/hub/servers/nats.ts +++ b/src/packages/hub/servers/nats.ts @@ -18,6 +18,11 @@ You could start this with import { createProxyServer } from "http-proxy"; import getLogger from "@cocalc/backend/logger"; +import { NATS_JWT_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; +import Cookies from "cookies"; +import { configureNatsUser, getJwt } from "@cocalc/server/nats/auth"; +import { type Router } from "express"; +import getAccount from "@cocalc/server/auth/get-account"; const logger = getLogger("hub:nats"); @@ -30,6 +35,37 @@ export async function proxyNatsWebsocket(req, socket, head) { const proxy = createProxyServer({ ws: true, target, + timeout: 5000, }); proxy.ws(req, socket, head); } + +export function initNatsServer(router: Router) { + router.get("/nats", async (req, res) => { + const account_id = await getAccount(req); + if (account_id) { + await setNatsCookie(req, res, account_id); + } else { + res.json({ error: "not signed in" }); + } + }); +} + +async function setNatsCookie(req, res, account_id: string) { + try { + const jwt = await getJwt({ account_id }); + // todo: how frequent? + await configureNatsUser({ account_id }); + const cookies = new Cookies(req, res, { secure: true }); + // 6 months -- long is fine now since we support "sign out everywhere" ? + const maxAge = 1000 * 24 * 3600 * 30 * 6; + cookies.set(NATS_JWT_COOKIE_NAME, jwt, { + maxAge, + sameSite: true, + }); + } catch (err) { + res.json({ error: `Problem setting cookie -- ${err.message}.` }); + return; + } + res.json({ account_id }); +} diff --git a/src/packages/next/pages/api/v2/auth/sign-in.ts b/src/packages/next/pages/api/v2/auth/sign-in.ts index 42fadfa1fe..8f6f81eca0 100644 --- a/src/packages/next/pages/api/v2/auth/sign-in.ts +++ b/src/packages/next/pages/api/v2/auth/sign-in.ts @@ -91,11 +91,6 @@ export async function signUserIn(req, res, account_id: string): Promise { maxAge: ttl_s * 1000, sameSite: samesite_remember_me, }); - // todo: NATS POC! - cookies.set( - "cocalc_nats_jwt_cookie", - "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiI1QjNKWVozNjJTUDdHUzZGU05JNzJRRFkyQVM1RUVYSUNERFNIM01GSkk3R0k2TTdRUUZBIiwiaWF0IjoxNzM3NDE1NTcxLCJpc3MiOiJBQjJLVEVGUFIyTzc2UE9aVVBZRVFTS1RaQVg2R0lOQVZUNkpXU0g2UUI3TENNNFhIRlRITVgyTCIsIm5hbWUiOiJ3c3RlaW4iLCJzdWIiOiJVQVFXUFc3QktRSkJLWUwzQUtUMjdIRE43UFpONEZJSlNUUDRCU0o1MktOUlhaM1ZLWk9QNk1DRiIsIm5hdHMiOnsicHViIjp7ImFsbG93IjpbIl9JTkJPWC5cdTAwM2UiLCJodWIuYXBpLjI3NWYxZGI3LWJmMzctNGI0NC1iOWFhLWQ2NDY5NDI2OWM5ZiJdfSwic3ViIjp7ImFsbG93IjpbIl9JTkJPWC5cdTAwM2UiLCJodWIuYXBpLjI3NWYxZGI3LWJmMzctNGI0NC1iOWFhLWQ2NDY5NDI2OWM5ZiJdfSwic3VicyI6LTEsImRhdGEiOi0xLCJwYXlsb2FkIjotMSwiYmVhcmVyX3Rva2VuIjp0cnVlLCJ0eXBlIjoidXNlciIsInZlcnNpb24iOjJ9fQ.JQBl6OoCCFmuxgoIgA7lrE_75Ut_MQ6hfu9W9Y7BW2Wvz-Bq7kxCZ_kKWhoLR8sH4RBQokwv48alGkXKzbnsAQ", - ); } catch (err) { res.json({ error: `Problem setting cookie -- ${err.message}.` }); return; diff --git a/src/packages/server/nats/api.ts b/src/packages/server/nats/api.ts index a5af8ce653..e85161c893 100644 --- a/src/packages/server/nats/api.ts +++ b/src/packages/server/nats/api.ts @@ -33,8 +33,10 @@ const logger = getLogger("server:nats:api"); const jc = JSONCodec(); export async function initAPI(nc) { - logger.debug("initAPI -- subject='hub.api', options=", { queue: "0" }); - const sub = nc.subscribe("hub.api.>", { queue: "0" }); + logger.debug("initAPI -- subject='hub.account.api', options=", { + queue: "0", + }); + const sub = nc.subscribe("hub.account.api.>", { queue: "0" }); for await (const mesg of sub) { handleApiRequest(mesg); } @@ -45,7 +47,7 @@ async function handleApiRequest(mesg) { let resp; try { const segments = mesg.subject.split("."); - const account_id = segments[2]; + const account_id = segments[3]; if (!isValidUUID(account_id)) { throw Error(`invalid account_id '${account_id}'`); } diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 702b6373ba..25c08ec1ce 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -23,7 +23,7 @@ const logger = getLogger("server:nats:auth"); export async function nsc(args: string[]) { // todo: for production we have to put some authentication // options, e.g., taken from the database. Skip that for now. - console.log(`nsc ${args.join(" ")}`); + // console.log(`nsc ${args.join(" ")}`); return await executeCode({ command: "nsc", args }); } @@ -95,7 +95,7 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { throw Error("must be a valid uuid"); } const userType = getCoCalcUserType(cocalcUser); - const goalPub = new Set([`hub.${userType}.${userId}.>`]); + const goalPub = new Set([`hub.${userType}.api.${userId}`]); const goalSub = new Set(["_INBOX.>"]); if (userType == "account") { diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts index e3291c38e1..b16fa3b7ca 100644 --- a/src/packages/server/nats/index.ts +++ b/src/packages/server/nats/index.ts @@ -5,7 +5,7 @@ import { initAPI } from "./api"; const logger = getLogger("server:nats"); const creds = `-----BEGIN NATS USER JWT----- -eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJNSEtXWTU0WDVNSUxXUEZBRkdRTFZVRkdTT0VCSDZUMjMyVjUzRzVRSjI3RFJWTFhOUk1BIiwiaWF0IjoxNzM3NDEwMTM4LCJpc3MiOiJBQjJLVEVGUFIyTzc2UE9aVVBZRVFTS1RaQVg2R0lOQVZUNkpXU0g2UUI3TENNNFhIRlRITVgyTCIsIm5hbWUiOiJodWIiLCJzdWIiOiJVQUhTQ1hVUVEzSFVVRlFIV0xUN0tYVDRXSU1ZSkdSQ1VLUUROWEZQRURCNU1WWkNMNkJKTldWVSIsIm5hdHMiOnsicHViIjp7ImFsbG93IjpbIl9JTkJPWC5cdTAwM2UiXX0sInN1YiI6eyJhbGxvdyI6WyJodWIuYXBpLlx1MDAzZSJdfSwic3VicyI6LTEsImRhdGEiOi0xLCJwYXlsb2FkIjotMSwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.3lQtNIOlj1nEStkbTSz4j25T1tIYfshWVvjyYE-ArkcmmCpzynoYCxEyKF8QuMNdZ30YTYu6xxcqluzjVCk_DA +eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJFNUhFT1g3VFJETVNCVzdWRUNTVkRDRVlZVkRON0lZNUMyVzZNWEw2RVRVQVJSNDVZTkhBIiwiaWF0IjoxNzM3NDkxNTMwLCJpc3MiOiJBQjJLVEVGUFIyTzc2UE9aVVBZRVFTS1RaQVg2R0lOQVZUNkpXU0g2UUI3TENNNFhIRlRITVgyTCIsIm5hbWUiOiJodWIiLCJzdWIiOiJVQUhTQ1hVUVEzSFVVRlFIV0xUN0tYVDRXSU1ZSkdSQ1VLUUROWEZQRURCNU1WWkNMNkJKTldWVSIsIm5hdHMiOnsicHViIjp7ImFsbG93IjpbIl9JTkJPWC5cdTAwM2UiXX0sInN1YiI6eyJhbGxvdyI6WyJodWIuXHUwMDNlIl19LCJzdWJzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJ0eXBlIjoidXNlciIsInZlcnNpb24iOjJ9fQ.yaXnOnTFqJQvTwkdWpRS9nQSSZJrUKJRqJYcqUj1ymz_eDdEZ-UdKrFCIFT7GSkbhIRlt5E6GCAeYZx2X9brCg ------END NATS USER JWT------ ************************* IMPORTANT ************************* From 38de8bd50b080f473fd2d1a3e72b857ad37596be Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 21 Jan 2025 22:45:51 +0000 Subject: [PATCH 011/281] nats: automate pushing/pulling local nsc with cluster --- src/packages/hub/servers/nats.ts | 2 +- .../next/pages/api/v2/accounts/sign-out.ts | 12 ++++++--- .../next/pages/api/v2/auth/sign-in.ts | 7 +++++- src/packages/server/nats/auth.ts | 25 +++++++++++++++++++ src/packages/util/db-schema/site-defaults.ts | 2 +- 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/src/packages/hub/servers/nats.ts b/src/packages/hub/servers/nats.ts index 11d67d0f99..c60dab7d93 100644 --- a/src/packages/hub/servers/nats.ts +++ b/src/packages/hub/servers/nats.ts @@ -35,7 +35,7 @@ export async function proxyNatsWebsocket(req, socket, head) { const proxy = createProxyServer({ ws: true, target, - timeout: 5000, + timeout: 3000, }); proxy.ws(req, socket, head); } diff --git a/src/packages/next/pages/api/v2/accounts/sign-out.ts b/src/packages/next/pages/api/v2/accounts/sign-out.ts index 1d9fdcaf5a..71baff5940 100644 --- a/src/packages/next/pages/api/v2/accounts/sign-out.ts +++ b/src/packages/next/pages/api/v2/accounts/sign-out.ts @@ -12,24 +12,27 @@ import { deleteAllRememberMe, } from "@cocalc/server/auth/remember-me"; import getParams from "lib/api/get-params"; - import { apiRoute, apiRouteOperation } from "lib/api"; import { SuccessStatus } from "lib/api/status"; import { AccountSignOutInputSchema, AccountSignOutOutputSchema, } from "lib/api/schema/accounts/sign-out"; +import { + NATS_JWT_COOKIE_NAME, + REMEMBER_ME_COOKIE_NAME, +} from "@cocalc/backend/auth/cookie-names"; async function handle(req, res) { try { - await signOut(req); + await signOut(req, res); res.json(SuccessStatus); } catch (err) { res.json({ error: err.message }); } } -async function signOut(req): Promise { +async function signOut(req, res): Promise { const { all } = getParams(req); if (all) { // invalidate all remember me cookies for this account. @@ -41,6 +44,9 @@ async function signOut(req): Promise { if (!hash) return; // not signed in await deleteRememberMe(hash); } + // also delete any security relevant cookies for safety and to avoid confusion. + res.clearCookie(NATS_JWT_COOKIE_NAME); + res.clearCookie(REMEMBER_ME_COOKIE_NAME); } export default apiRoute({ diff --git a/src/packages/next/pages/api/v2/auth/sign-in.ts b/src/packages/next/pages/api/v2/auth/sign-in.ts index 8f6f81eca0..7162f73ed3 100644 --- a/src/packages/next/pages/api/v2/auth/sign-in.ts +++ b/src/packages/next/pages/api/v2/auth/sign-in.ts @@ -18,7 +18,10 @@ Sign in works as follows: import getPool from "@cocalc/database/pool"; import { createRememberMeCookie } from "@cocalc/server/auth/remember-me"; -import { REMEMBER_ME_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; +import { + NATS_JWT_COOKIE_NAME, + REMEMBER_ME_COOKIE_NAME, +} from "@cocalc/backend/auth/cookie-names"; import { recordFail, signInCheck } from "@cocalc/server/auth/throttle"; import Cookies from "cookies"; import getParams from "lib/api/get-params"; @@ -91,6 +94,8 @@ export async function signUserIn(req, res, account_id: string): Promise { maxAge: ttl_s * 1000, sameSite: samesite_remember_me, }); + // ensure there is no stale JWT cookie + res.clearCookie(NATS_JWT_COOKIE_NAME); } catch (err) { res.json({ error: `Problem setting cookie -- ${err.message}.` }); return; diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 25c08ec1ce..76db16510f 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -17,6 +17,8 @@ import { executeCode } from "@cocalc/backend/execute-code"; import getPool from "@cocalc/database/pool"; import { isValidUUID } from "@cocalc/util/misc"; import getLogger from "@cocalc/backend/logger"; +import { throttle } from "lodash"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; const logger = getLogger("server:nats:auth"); @@ -154,12 +156,14 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { } } const args = ["edit", "signing-key", "--sk", name]; + let changed = false; if (rm.length > 0) { // --rm applies to both pub and sub and happens after adding, // so we have to do it separately at the beginning in order to // handle some edge cases (that might never happen). logger.debug("configureNatsUser ", { rm }); await nsc([...args, "--rm", rm.join(",")]); + changed = true; } if (sub.length > 0 || pub.length > 0) { if (sub.length > 0) { @@ -172,6 +176,10 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { } logger.debug("configureNatsUser ", { pub, sub }); await nsc(args); + changed = true; + } + if (changed) { + pushToServer(); } } @@ -211,7 +219,23 @@ export async function getScopedSigningKey(natsUser: string) { return obj; } +// we push to server whenever there's a change, but at most once every few seconds, +// and if we push while a push is happening, it doesn't do it twice at once. +export const pushToServer = throttle( + reuseInFlight(async () => { + try { + await nsc(["push", "-A"]); + } catch (err) { + // TODO: adminNotification? This could be very serious. + logger.debug("push configuration to nats server failed", err); + } + }), + 3000, + { leading: true, trailing: true }, +); + export async function createNatsUser(cocalcUser: CoCalcUser) { + await nsc(["pull", "-A"]); const { stderr } = await nsc(["edit", "account", "--sk", "generate"]); const key = stderr.trim().split('"')[1]; const name = getNatsUserName(cocalcUser); @@ -221,6 +245,7 @@ export async function createNatsUser(cocalcUser: CoCalcUser) { await nsc(["edit", "signing-key", "--sk", key, "--role", name, "--bearer"]); await nsc(["add", "user", name, "--private-key", name]); await configureNatsUser(cocalcUser); + pushToServer(); } export async function getJwt(cocalcUser: CoCalcUser): Promise { diff --git a/src/packages/util/db-schema/site-defaults.ts b/src/packages/util/db-schema/site-defaults.ts index 0f3d7bca7e..efa43cdbb4 100644 --- a/src/packages/util/db-schema/site-defaults.ts +++ b/src/packages/util/db-schema/site-defaults.ts @@ -364,7 +364,7 @@ export const site_settings_conf: SiteSettings = { // ========= THEMING =============== dns: { name: "Domain name", - desc: "DNS for your server, e.g. `cocalc.universe.edu`. Does NOT include the basePath. It optionally can start with `http://` (for non SSL) and end in a `:number` for a port. This is mainly used for password resets and invitation and sign up emails, since they need to know a link to the site.", + desc: "DNS for your server, e.g. `cocalc.universe.edu`. **Do NOT include the basePath or the https:// prefix.** It optionally can start with `http://` (for non SSL) and end in a `:number` for a port. This is mainly used for password resets and invitation and sign up emails, since they need to know a link to the site.", default: "", to_val: to_trimmed_str, //valid: valid_dns_name, From 42c8864ef36adcd27075f304f17258676a1e23fa Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 21 Jan 2025 23:59:22 +0000 Subject: [PATCH 012/281] nats: provide JWT to project, let project connect, get it to do something useful --- .../nats/notes.md => docs/nats/devlog.md | 51 +- src/packages/frontend/client/nats.ts | 3 +- src/packages/project/info-json.ts | 1 - src/packages/project/nats/index.ts | 89 ++ src/packages/project/package.json | 2 +- src/packages/project/servers/nats/index.ts | 61 -- src/packages/project/servers/secret-token.ts | 2 - src/packages/server/nats/auth.ts | 8 +- src/packages/server/projects/control/util.ts | 2 + src/scripts/storage_gluster.py | 800 ------------------ 10 files changed, 146 insertions(+), 873 deletions(-) rename src/packages/server/nats/notes.md => docs/nats/devlog.md (62%) create mode 100644 src/packages/project/nats/index.ts delete mode 100644 src/packages/project/servers/nats/index.ts delete mode 100755 src/scripts/storage_gluster.py diff --git a/src/packages/server/nats/notes.md b/docs/nats/devlog.md similarity index 62% rename from src/packages/server/nats/notes.md rename to docs/nats/devlog.md index 065874064b..cc60427f6f 100644 --- a/src/packages/server/nats/notes.md +++ b/docs/nats/devlog.md @@ -1,4 +1,4 @@ -# Systematically keep track of NATS experiment here! +# NATS Development and Integration Log ## [x] Goal: nats from nodejs @@ -105,7 +105,7 @@ Creating a creds file that encodes a JWT that says what you can publish and subs - make it so user with account\_id can publish to hub.api.{account\_id} makes it so we know the account\_id automatically by virtue of what was published to. This works. -## [ ] Goal: Solve Critical Auth Problems +## [x] Goal: Solve Critical Auth Problems Now need to solve two problems: @@ -115,9 +115,50 @@ I finally figured this out after WASTING a lot of time with stupid AI misleading This makes it so the server doesn't check the signature of the JWT against the _user_ . Putting exactly the JWT token string in the cookie then works because "bearer" literally tells the backend server not to do the signature check. I think this is secure and the right approach because the server checks that the JWT is valid using the account and operator signatures. -- [ ] GOAL: automate creation of creds for browser clients, i.e., what we just did with the nsc tool. +**WAIT!** Using signing keys [https://docs.nats.io/using\-nats/nats\-tools/nsc/signing\_keys](https://docs.nats.io/using-nats/nats-tools/nsc/signing_keys) \(and https://youtu.be/KmGtnFxHnVA?si=0uvLMBTJ5TUpem4O \) is VASTLY superior. There's just one JWT issued to each user, and we make a server\-side\-only JWT for their account that has everything. The user never has to reconnect or change their JWT. We can adjust the subject on the fly to account for running projects \(or collaboration changes\) at any time server side. Also the size limits go away, so we don't have to compress project\_id's \(probably\). -If we can do the above, we should also be able to do something similar for: +## Goal: Implement Auth Solution for Browsers -- [ ] GOAL: connecting to projects. I.e., access to a project means you can publish to `projects.{project_id}.>` Also, projects should have access to something under hub. +- [x] automate creation of creds for browser clients, i.e., what we just did with the nsc tool manually +- + +--- + +This is my top priority goal for NOW! + +What's the plan? + +Need to figure out how to do all the nsc stuff from javascript, storing results in the database? + +- Question: how do we manage creating signing keys and users from nodejs? Answer: clear from many sources that we must use the nsc CLI tool via subprocess calls. Seems fine to me. +- [x] When a user signs in, we check for their JWT in the database. If it is there, set the cookie. If not, create the signing key and JWT for them, save in database, and set the cookie. +- [x] update nats\-server resolver state after modifying signing cookie's subjects configuration. + +``` +nsc edit operator --account-jwt-server-url nats://localhost:4222 +``` + +Now I can do `nsc push` and it just works. + +[x] TODO: when signing out, need to delete the jwt cookie or dangerous private info leaks... and also new info not set properly. + +- [x] similar creds for projects, I.e., access to a project means you can publish to `projects.{project_id}.>` Also, projects should have access to something under hub. + +## Goal: Auth for Projects + + + + + + +Some thoughts about project auth security: + +- [ ] when collaborators on a project leave maybe we change JWT? Otherwise, in theory any user of a project can probably somehow get access to the project's JWT \(it's in memory at least\) and still act as the project. Changing JWT requires reconnect. This could be "for later", since even now we don't have this level of security! +- [ ] restarting project could change JWT. That's like the current project's secret token being changed. + +--- + +## Goal: nats-server automation of creation and configuration of system account, operator, etc. + +- This looks helpful: https://www.synadia.com/newsletter/nats-weekly-27/ diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index ed3998fc1d..73b0c6f343 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -70,8 +70,9 @@ export class NatsClient { params?: object; }) => { const c = await this.getConnection(); + const subject = `project.${project_id}.owner.${this.client.account_id}.api`; const resp = await c.request( - `projects.${project_id}.api`, + subject, this.jc.encode({ endpoint, params, diff --git a/src/packages/project/info-json.ts b/src/packages/project/info-json.ts index 807fd2f266..f986022025 100644 --- a/src/packages/project/info-json.ts +++ b/src/packages/project/info-json.ts @@ -7,7 +7,6 @@ about the project. import { writeFile } from "node:fs/promises"; import { networkInterfaces } from "node:os"; - import basePath from "@cocalc/backend/base-path"; import { infoJson, project_id, username } from "./data"; import { getOptions } from "./init-program"; diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts new file mode 100644 index 0000000000..c3793f1b9a --- /dev/null +++ b/src/packages/project/nats/index.ts @@ -0,0 +1,89 @@ +/* +How to do development (so in a dev project doing cc-in-cc dev). + +1. Open a terminal in the project itself, which sets up the required environment variables, e.g., + - COCALC_NATS_JWT -- this has the valid JWT issued to grant the project rights to use nats + - COCALC_PROJECT_ID + +2. cd to your dev packages/project source code, e.g., ../cocalc/src/packages/project + +3. Do this: + + echo 'require("@cocalc/project/nats").default()' | DEBUG=cocalc:* DEBUG_CONSOLE=yes node + +4. Use the browser to see the project is on nats and works: + + await cc.client.nats_client.project({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e', endpoint:"exec", params:{command:'echo $COCALC_PROJECT_ID'}}) + +*/ + +import { getLogger } from "@cocalc/project/logger"; +import { connect, JSONCodec, jwtAuthenticator } from "nats"; +import { project_id } from "@cocalc/project/data"; + +const logger = getLogger("server:nats"); + +export default async function initNatsServer() { + logger.debug("initializing nats cocalc project server"); + if (!process.env.COCALC_NATS_JWT) { + throw Error("environment variable COCALC_NATS_JWT *must* be set"); + } + const nc = await connect({ + authenticator: jwtAuthenticator(process.env.COCALC_NATS_JWT), + }); + logger.debug(`connected to ${nc.getServer()}`); + initAPI(nc); +} + +const jc = JSONCodec(); + +export async function initAPI(nc) { + const subject = `project.${project_id}.>`; + logger.debug(`initAPI -- NATS project subject '${subject}'`); + const sub = nc.subscribe(subject); + for await (const mesg of sub) { + handleRequest(mesg); + } +} + +async function handleRequest(mesg) { + const segments = mesg.subject.split("."); + const group = segments[2]; // 'owner', 'collaborator', etc. + const account_id = segments[3]; + const type = segments[4]; // e.g., 'api', etc. + if (type == "api") { + await handleApiRequest({ mesg, group, account_id }); + } else { + logger.debug(`unknown request type '${type}'`); + } +} + +async function handleApiRequest({ mesg, group, account_id }) { + const request = jc.decode(mesg.data) ?? {}; + let resp; + try { + const { endpoint, params } = request as any; + logger.debug("handling project request:", { endpoint, group, account_id }); + resp = await getResponse({ endpoint, params }); + } catch (err) { + resp = { error: `${err}` }; + } + logger.debug("responding with ", resp); + mesg.respond(jc.encode(resp)); +} + +import { handleExecShellCode } from "@cocalc/project/exec_shell_code"; +import { realpath } from "@cocalc/project/browser-websocket/realpath"; + +async function getResponse({ endpoint, params }) { + switch (endpoint) { + case "ping": + return { pong: Date.now() }; + case "realpath": + return realpath(params.path); + case "exec": + return await handleExecShellCode(params); + default: + throw Error(`unknown endpoint '${endpoint}'`); + } +} diff --git a/src/packages/project/package.json b/src/packages/project/package.json index e8defd994e..0d08e2d19b 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -4,7 +4,7 @@ "description": "CoCalc: project daemon", "exports": { "./named-servers": "./dist/named-servers/index.js", - "./servers/nats": "./dist/servers/nats/index.js", + "./nats": "./dist/nats/index.js", "./*": "./dist/*.js" }, "keywords": [ diff --git a/src/packages/project/servers/nats/index.ts b/src/packages/project/servers/nats/index.ts deleted file mode 100644 index 63c5a733e1..0000000000 --- a/src/packages/project/servers/nats/index.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* -Run for a project_id you want to simulate: - - export HOME=... # (optional) - export COCALC_PROJECT_ID='81e0c408-ac65-4114-bad5-5f4b6539bd0e' - echo 'require("@cocalc/project/servers/nats").default()' | node - -then in the browser: - - await cc.client.nats_client.project({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e', endpoint:"exec", params:{command:'echo $COCALC_PROJECT_ID'}}) - -*/ - -import { getLogger } from "@cocalc/project/logger"; -import { connect, JSONCodec } from "nats"; -import { project_id } from "@cocalc/project/data"; - -const logger = getLogger("server:nats"); - -export default async function initNatsServer() { - logger.debug("initializing nats cocalc project server"); - const nc = await connect(); - logger.debug(`connected to ${nc.getServer()}`); - initAPI(nc); -} - -const jc = JSONCodec(); - -export async function initAPI(nc) { - const subject = `projects.${project_id}.api`; - logger.debug(`initAPI -- NATS project subject '${subject}'`); - const sub = nc.subscribe(subject); - for await (const mesg of sub) { - handleApiRequest(mesg); - } -} - -async function handleApiRequest(mesg) { - const request = jc.decode(mesg.data) ?? {}; - let resp; - try { - // TODO: obviously user-provided account_id is no good! This is a POC. - const { endpoint, params } = request as any; - logger.debug("handling project request:", { endpoint }); - resp = await getResponse({ endpoint, params }); - } catch (err) { - resp = { error: `${err}` }; - } - mesg.respond(jc.encode(resp)); -} - -import { handleExecShellCode } from "@cocalc/project/exec_shell_code"; - -async function getResponse({ endpoint, params }) { - switch (endpoint) { - case "exec": - return await handleExecShellCode(params); - default: - throw Error(`unknown endpoint '${endpoint}'`); - } -} diff --git a/src/packages/project/servers/secret-token.ts b/src/packages/project/servers/secret-token.ts index cfb8ad91fe..2db578bf56 100644 --- a/src/packages/project/servers/secret-token.ts +++ b/src/packages/project/servers/secret-token.ts @@ -10,9 +10,7 @@ Generate the "secret_token" file if it does not already exist. import { callback } from "awaiting"; import { randomBytes } from "crypto"; import { chmod, readFile, writeFile } from "node:fs/promises"; - import { secretToken as secretTokenPath } from "@cocalc/project/data"; - import { getLogger } from "@cocalc/project/logger"; const winston = getLogger("secret-token"); diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 76db16510f..fcfcc795d5 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -97,7 +97,7 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { throw Error("must be a valid uuid"); } const userType = getCoCalcUserType(cocalcUser); - const goalPub = new Set([`hub.${userType}.api.${userId}`]); + const goalPub = new Set(["_INBOX.>", `hub.${userType}.api.${userId}`]); const goalSub = new Set(["_INBOX.>"]); if (userType == "account") { @@ -111,6 +111,10 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { // TODO: there will be other subjects // TODO: something similar for projects, e.g., they can publish to a channel that browser clients // will listen to, e.g., for timetravel editing. + } else if (userType == "project") { + // the project can publish to anything under its own subject: + goalPub.add(`project.${userId}.>`); + goalSub.add(`project.${userId}.>`); } // **Subject Permissions SYNC Algorithm ** @@ -224,7 +228,7 @@ export async function getScopedSigningKey(natsUser: string) { export const pushToServer = throttle( reuseInFlight(async () => { try { - await nsc(["push", "-A"]); + await nsc(["push", "-a", "SYS"]); } catch (err) { // TODO: adminNotification? This could be very serious. logger.debug("push configuration to nats server failed", err); diff --git a/src/packages/server/projects/control/util.ts b/src/packages/server/projects/control/util.ts index 9d8df0c10a..d76b0f451b 100644 --- a/src/packages/server/projects/control/util.ts +++ b/src/packages/server/projects/control/util.ts @@ -14,6 +14,7 @@ import base_path from "@cocalc/backend/base-path"; import { db } from "@cocalc/database"; import { getProject } from "."; import { pidFilename, pidUpdateIntervalMs } from "@cocalc/util/project-info"; +import { getJwt } from "@cocalc/server/nats/auth"; const logger = getLogger("project-control:util"); @@ -275,6 +276,7 @@ export async function getEnvironment( USER, COCALC_EXTRA_ENV: extra_env, PATH: `${HOME}/bin:${HOME}/.local/bin:${process.env.PATH}`, + COCALC_NATS_JWT: await getJwt({ project_id }), }, }; } diff --git a/src/scripts/storage_gluster.py b/src/scripts/storage_gluster.py deleted file mode 100755 index 8bb797a004..0000000000 --- a/src/scripts/storage_gluster.py +++ /dev/null @@ -1,800 +0,0 @@ -#!/usr/bin/env python -############################################################################### -# -# CoCalc: Collaborative Calculation -# -# Copyright (C) 2016, Sagemath Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -############################################################################### - -import argparse, cPickle, hashlib, json, logging, os, sys, time, random -from uuid import UUID - -log = None - -# This is so we can import salvus/salvus/daemon.py -sys.path.append('/home/salvus/salvus/salvus/') - - -def check_uuid(uuid): - if UUID(uuid).version != 4: - raise RuntimeError("invalid uuid") - - -def uid(uuid): - # We take the sha-512 of the uuid just to make it harder to force a collision. Thus even if a - # user could somehow generate an account id of their choosing, this wouldn't help them get the - # same uid as another user. - n = hash(hashlib.sha512(uuid).digest()) % ( - 4294967294 - 1000 - ) # 2^32-2=max uid, as keith determined by a program + experimentation. - return n + 1001 - - -def cmd(s, exit_on_error=True): - log.debug(s) - #s += ' &>/dev/null' - t = time.time() - if os.system(s): - if exit_on_error: - raise RuntimeError("Error running '%s'" % s) - log.debug("time: %s seconds" % (time.time() - t)) - - -def cmd2(s): - log.debug(s) - from subprocess import Popen, PIPE - out = Popen( - s, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=not isinstance(s, list)) - e = out.wait() - x = out.stdout.read() + out.stderr.read() - log.debug(x) - return x, e - - -def path_to_project(storage, project_id): - return os.path.join(storage, project_id[:2], project_id[2:4], project_id) - - -def migrate_project_to_storage(src, storage, min_size_mb, max_size_mb, - new_only): - info_json = os.path.join(src, '.sagemathcloud', 'info.json') - if not os.path.exists(info_json): - log.debug("Skipping since %s does not exist" % info_json) - return - project_id = json.loads(open(info_json).read())['project_id'] - projectid = project_id.replace('-', '') - target = path_to_project(storage, project_id) - try: - if os.path.exists(target): - if new_only: - log.debug( - "skipping %s (%s) since it already exists (and new_only=True)" - % (src, project_id)) - return - mount_project(storage=storage, project_id=project_id, force=False) - else: - # create - os.makedirs(target) - os.chdir(target) - current_size_mb = int( - os.popen("du -s '%s'" % src).read().split()[0]) // 1000 + 1 - size = min(max_size_mb, max(min_size_mb, current_size_mb)) - - # Using many small img files might seem like a good idea. It isn't, since mount takes massively longer, etc. - #img_size_mb = 128 - #images = ['%s/%s.img'%(target, i) for i in range(size//img_size_mb + 1)] - #for img in images: - # cmd("truncate -s %sM %s"%(img_size_mb,img)) - #images = ' '.join(images) - - images = '%s/%s.img' % (target, 0) - cmd("truncate -s %sM %s" % (size, images)) - - cmd("zpool create -m /home/%s project-%s %s" % - (projectid, project_id, images)) - cmd("zfs set compression=gzip project-%s" % project_id) - cmd("zfs set dedup=on project-%s" % project_id) - cmd("zfs set snapdir=visible project-%s" % project_id) - - # rsync data over - double_verbose = False - cmd("time rsync -axH%s --delete --exclude .forever --exclude .bup %s/ /home/%s/" - % ('v' if double_verbose else '', src, projectid), - exit_on_error=False) - id = uid(project_id) - cmd("chown %s:%s -R /home/%s/" % (id, id, projectid)) - cmd("df -h /home/%s; zfs get compressratio project-%s; zpool get dedupratio project-%s" - % (projectid, project_id, project_id)) - finally: - unmount_project(project_id=project_id) - - -def mount_project(storage, project_id, force): - check_uuid(project_id) - id = uid(project_id) - target = path_to_project(storage, project_id) - out, e = cmd2("zpool import %s project-%s -d %s" % ('-f' if force else '', - project_id, target)) - if e: - if 'a pool with that name is already created' in out: - # no problem - pass - else: - print "could not get pool" - sys.exit(1) - projectid = project_id.replace('-', '') - # the -o makes it so in the incredibly unlikely event of a collision, no big deal. - cmd("groupadd -g %s -o %s" % (id, projectid), exit_on_error=False) - cmd("useradd -u %s -g %s -o -d /home/%s/ %s" % (id, id, projectid, - projectid), - exit_on_error=False) # error if user already exists is fine. - - -def unmount_project(project_id): - check_uuid(project_id) - projectid = project_id.replace('-', '') - cmd("pkill -9 -u %s" % projectid, exit_on_error=False) - cmd("deluser --force %s" % projectid, exit_on_error=False) - time.sleep(.5) - out, e = cmd2("zpool export project-%s" % project_id) - if e: - if 'no such pool' not in out: - # not just a problem due to pool not being mounted. - print "Error unmounting pool -- %s" % out - sys.exit(1) - - -def tinc_address(): - return os.popen('ifconfig tun0|grep "inet addr"').read().split()[1].split( - ':')[1].strip() - - -def info_json(path): - if not os.path.exists('locations.dat'): - sys.stderr.write( - 'Please run this from a node with db access to create locations.dat\n\t\techo "select location,project_id from projects limit 30000;" | cqlsh_connect 10.1.3.2 |grep "{" > locations.dat' - ) - sys.exit(1) - db = {} - host = tinc_address() - log.info("parsing database...") - for x in open('locations.dat').readlines(): - if x.strip(): - location, project_id = x.split('|') - location = json.loads(location.strip()) - project_id = project_id.strip() - if location['host'] == host: - if location['username'] in db: - log.warning("WARNING: collision -- %s, %s" % (location, - project_id)) - db[location['username']] = { - 'location': location, - 'project_id': project_id, - 'base_url': '' - } - v = [os.path.abspath(x) for x in path] - for i, path in enumerate(v): - log.info("** %s of %s" % (i + 1, len(v))) - SMC = os.path.join(path, '.sagemathcloud') - if not os.path.exists(SMC): - log.warning( - "Skipping '%s' since no .sagemathcloud directory" % path) - continue - f = os.path.join(path, '.sagemathcloud', 'info.json') - username = os.path.split(path)[-1] - if not os.path.exists(f): - if username not in db: - log.warning("Skipping '%s' since not in database!" % username) - else: - s = json.dumps(db[username], separators=(',', ':')) - log.info("writing '%s': '%s'" % (f, s)) - open(f, 'w').write(s) - os.system('chmod a+rw %s' % f) - - -def modtime(f): - try: - return int(os.stat(f).st_mtime) - except: - return 0 # 1970... - - -def copy_file_efficiently(src, dest): - """ - Copy a possibly sparse file from a brick to a mounted glusterfs volume, if the dest is older. - - This for now -- later we might use a different method when the file is above a certain - size threshold (?). However, I can't think of any possible better method, really; anything - involving computing a diff between the two files would require *reading* them, so already - takes way too long (in sharp contrast to the ever-clever bup, which uses a blum filter!). - - This will raise a RuntimeError if something goes wrong. - """ - import uuid - s0, s1 = os.path.split(dest) - if s1.startswith('.glusterfs'): - # never copy around/sync any of the temp files we create below. - return - - # The clock of the destination is used when doing this copy, so it's - # *critical* that the clocks be in sync. Run ntp!!!!! - dest_modtime = modtime(dest) - if dest_modtime >= modtime(src): - return - - if not os.path.exists(s0): - os.makedirs(s0) - lock = os.path.join(s0, ".glusterfs-lock-%s" % s1) - dest0 = os.path.join(s0, ".glusterfs-tmp-%s-%s" % (str(uuid.uuid4()), s1)) - - now = time.time() - recent = now - 5 * 60 # recent time = 5 minutes ago - if os.path.exists(lock): - log.debug( - "another daemon is either copying the same file right now (or died)." - ) - # If mod time of the lock is recent, just give up. - t = modtime(lock) - if t >= recent: - return # recent lock - # check that dest0 exists and has mod time < 5 minutes; otherwise, take control. - if os.path.exists(dest0) and modtime(dest0) >= recent: - return - - if os.stat(src).st_mode == 33280: - log.info( - "skipping copy since source '%s' suddenly became special link file", - src) - return - - log.info("sync: %s --> %s" % (src, dest)) - t = time.time() - try: - log.info(cmd2('ls -lhs "%s"' % src)[0]) - cmd("touch '%s'; cp -av '%s' '%s'" % (lock, src, dest0), - exit_on_error=True) - # check that modtime of dest is *still* older, i.e., that somehow somebody didn't - # just step in and change it. - if modtime(dest) == dest_modtime: - # modtime was unchanged. - cmd("mv -v '%s' '%s'" % (dest0, dest), exit_on_error=True) - - finally: - # remove the tmp file instead of leaving it there all corrupted. - if os.path.exists(dest0): - try: - os.unlink(dest0) - except: - pass - if os.path.exists(lock): - try: - os.unlink(lock) - except: - pass - - total_time = time.time() - t - log.info("time: %s" % total_time) - return total_time - - -def sync(src, dest): - """ - copy all older files from src/ to dest/. - - -- src/ = underyling *brick* path for some glusterfs host - -- dest/ = remote mounted glusterfs filesystem - """ - src = os.path.abspath(src) - dest = os.path.abspath(dest) - - cache_file = "/var/lib/glusterd/glustersync/cache.pickle" - if not os.path.exists("/var/lib/glusterd/glustersync"): - os.makedirs("/var/lib/glusterd/glustersync") - if os.path.exists(cache_file): - cache_all = cPickle.loads(open(cache_file).read()) - else: - cache_all = {} - if dest not in cache_all: - cache_all[dest] = {} - cache = cache_all[dest] - - log.info("sync: '%s' --> '%s'" % (src, dest)) - - import stat - - def walktree(top): - #log.info("scanning '%s'", top) - v = os.listdir(top) - random.shuffle(v) - for i, f in enumerate(v): - if f == '.glusterfs': - # skip the glusterfs meta-data - continue - if len(v) > 10: - log.debug("%s(%s/%s): %s", top, i + 1, len(v), f) - pathname = os.path.join(top, f) - - src_name = os.path.join(src, pathname) - dest_name = os.path.join(dest, pathname) - - st = os.stat(src_name) - - if st.st_mode == 33280: - # glusterfs meta-info file to indicate a moved file... - continue - - if stat.S_ISDIR(st.st_mode): - # It's a directory: create in target if necessary, then recurse... - ## !! we skip creation; this is potentially expensive and isn't needed for our application. - ##if not os.path.exists(dest_name): - ## try: - ## os.makedirs(dest_name) - ## except OSError: - ## if not os.path.exists(dest_name): - ## raise RuntimeError("unable to make directory '%s'"%dest_name) - try: - walktree(pathname) - except OSError, mesg: - log.warning("error walking '%s': %s", pathname, mesg) - - elif stat.S_ISREG(st.st_mode): - mtime = int(st.st_mtime) - if cache.get(src_name, {'mtime': 0})['mtime'] >= mtime: - continue - try: - copy_file_efficiently( - src_name, dest_name - ) # checks dest modtime before actually doing copy. - cache[src_name] = { - 'mtime': mtime, - 'size_mb': st.st_blocks // 2000 - } - except RuntimeError, mesg: - log.warning("error copying %s to %s; skipping.", src_name, - dest_name) - - else: - # Unknown file type, print a message - log.warning("unknown file type: %s", pathname) - - os.chdir(src) - walktree('.') - - s = cPickle.dumps(cache_all) - open(cache_file, 'w').write(s) - - -def sync_watch(sources, dests, min_sync_time): - ### WARNING -- this code does not work very well, and is sort of pointless. AVOID! - """ - Watch filesystem trees and on modification or creation, cp file, possibly creating directories. - The frequency of copying is limited in various ways. - - This uses inotify so that it is event driven. You must increase the number of watched files - that are allowed! "sudo sysctl fs.inotify.max_user_watches=10000000" and in /etc/sysctl.conf: - fs.inotify.max_user_watches=10000000 - - - sources = list of underyling *brick* path for some glusterfs host - - dests = list of paths of remote mounted glusterfs filesystems - - min_sync_time = never sync a file more frequently than this many seconds; no matter what, we - also wait at least twice the time it takes to sync out the file before syncing it again. - """ - sources = [os.path.abspath(src) for src in sources] - dests = [os.path.abspath(dest) for dest in dests] - - next_sync = {} # soonest time when may again sync a given file - - modified_files = set([]) - received_files = set([]) - - def add(pathname): - try: - if os.stat(pathname).st_mode == 33280: - # ignore gluster special files - log.debug("ignoring gluster special file: '%s'", pathname) - return - except: - pass - log.debug("inotify: %s" % pathname) - s = os.path.split(pathname) - if s[1].startswith('.glusterfs-lock-'): - received_files.add( - os.path.join(s[0], s[1][len('.glusterfs-lock-'):])) - elif s[1].startswith('.glusterfs'): - return - elif os.path.isfile(pathname): - modified_files.add(pathname) - - def handle_modified_files(): - if not modified_files: - return - log.debug("handling modified_files=%s", modified_files) - log.debug("received_files=%s", received_files) - now = time.time() - do_later = [] - for path in modified_files: - if path in sources: # ignore changes to the sources directories - continue - if path in received_files: # recently copied to us. - continue - if path not in next_sync or now >= next_sync[path]: - src = None - for s in sources: - if path.startswith(s): - src = s - break - if not src: - log.warning( - "not copying '%s' -- must be under a source: %s" % - (path, sources)) - continue - t0 = time.time() - for dest in dests: - dest_path = os.path.join(dest, path[len(src) + 1:]) - log.info("copy('%s', '%s')" % (path, dest_path)) - try: - copy_file_efficiently(path, dest_path) - except Exception, msg: - log.warning("problem syncing %s to %s! -- %s" % - (path, dest_path, msg)) - # no matter what, we wait at least twice the time (from now) that it takes to sync out the file before syncing it again. - next_sync[path] = time.time() + max(2 * (time.time() - t0), - min_sync_time) - else: - pass - #log.debug("waiting until later to sync (too frequent): '%s' "%path) - do_later.append(path) - modified_files.clear() - received_files.clear() - modified_files.update(do_later) - - import pyinotify - wm = pyinotify.WatchManager() # Watch Manager - mask = pyinotify.IN_CREATE | pyinotify.IN_MOVED_TO | pyinotify.IN_MODIFY | pyinotify.IN_CLOSE_WRITE - - class EventHandler(pyinotify.ProcessEvent): - def process_IN_CREATE(self, event): - print "Creating:", event.pathname - if os.path.isdir(event.pathname): - # created a directory -- add it to the watch list - watchers.append(wm.add_watch(event.pathname, mask)) - add(event.pathname) - - def process_IN_MOVED_TO(self, event): - print "File moved to:", event.pathname - add(event.pathname) - - def process_IN_MODIFY(self, event): - print "Modified:", event.pathname - add(event.pathname) - - def process_IN_CLOSE_WRITE(self, event): - print "Close write:", event.pathname - add(event.pathname) - - handler = EventHandler() - - # we get inotify events for *at most* timeout seconds, then handle them all - notifier = pyinotify.Notifier(wm, handler, timeout=1) - - t = time.time() - - watchers = [] - for src in sources: - log.info("adding watches to '%s' (this could take several minutes)..." - % src) - dot_gluster = os.path.join(src, '.glusterfs') - watchers.append( - wm.add_watch( - src, - mask, - rec=True, - exclude_filter=pyinotify.ExcludeFilter(['^' + dot_gluster]))) - log.info("watch added (%s seconds): listening for file events..." % - (time.time() - t)) - - def check_for_events(): - #print "check_for_events" - notifier.process_events() - while notifier.check_events( - ): #loop in case more events appear while we are processing - notifier.read_events() - notifier.process_events() - - while True: - check_for_events() - handle_modified_files() - time.sleep(1) - - -def volume_info(): - # parse 'gluster volume info' as a python object. - s, e = cmd2('unset PYTHONPATH; unset PYTHONHOME; gluster volume info') - if e: - raise RuntimeError(e) - v = {} - for x in s.split("\nVolume Name: "): - z = x.strip().splitlines() - if z: - name = z[0] - m = {'bricks': []} - for k in z[1:]: - i = k.find(':') - if i == -1: - continue - key = k[:i].strip() - val = k[i + 1:].strip() - if val: - if key.startswith('Brick'): - m['bricks'].append(val) - else: - m[key] = val - v[name] = m - return v - - -def ip_address(dest): - # get the ip address that is used to communicate with the given destination - import misc - return misc.local_ip_address(dest) - - -def mount_target_volumes(volume_name): - info = volume_info() - dests = [] - ip = None - mount = cmd2('mount')[0] - for name, data in volume_info().iteritems(): - if name.startswith('dc'): - v = name.split('-') - if len(v) >= 2 and v[1] == volume_name: - use = True - for brick in data['bricks']: - brick_ip, path = brick.split(':') - if ip_address(brick_ip) == brick_ip: - # this volume is partly hosted on this computer, hence not a target. - use = False - break - if use: - # ensure volume is mounted and add to list - if 'mnt/%s' % name not in mount: - cmd("mkdir -p '/mnt/%s'; mount -t glusterfs localhost:'/%s' '/mnt/%s'" - % (name, name, name)) - dests.append('/mnt/%s' % name) - return dests - - -def find_bricks(volume_name): - bricks = [] - ip = None - for name, data in volume_info().iteritems(): - if name.startswith('dc'): - v = name.split('-') - if len(v) >= 2 and v[1] == volume_name: - for brick in data['bricks']: - brick_ip, path = brick.split(':') - if ip_address(brick_ip) == brick_ip: - bricks.append(path) - return bricks - - -def setup_log(loglevel='DEBUG', logfile=''): - logging.basicConfig() - global log - log = logging.getLogger('storage') - if loglevel: - level = getattr(logging, loglevel.upper()) - log.setLevel(level) - - if logfile: - log.addHandler(logging.FileHandler(logfile)) - - log.info("logger started") - - -if __name__ == "__main__": - - parser = argparse.ArgumentParser(description="Project storage") - parser.add_argument( - "--loglevel", - dest='loglevel', - type=str, - default='INFO', - help="log level: useful options include INFO, WARNING and DEBUG") - parser.add_argument( - "--logfile", - dest="logfile", - type=str, - default='', - help="store log in this file (default: '' = don't log to a file)") - - subparsers = parser.add_subparsers(help='sub-command help') - - def migrate(args): - if not args.storage: - args.storage = os.environ['SALVUS_STORAGE'] - v = [os.path.abspath(x) for x in args.src] - for i, src in enumerate(v): - log.info("\n** %s of %s" % (i + 1, len(v))) - migrate_project_to_storage( - src=src, - storage=args.storage, - min_size_mb=args.min_size_mb, - max_size_mb=10000, - new_only=args.new_only) - - parser_migrate = subparsers.add_parser( - 'migrate', help='migrate to or update project in storage pool') - parser_migrate.add_argument( - "--storage", - help= - "the directory where project image directories are stored (default: $SALVUS_STORAGE enviro var)", - type=str, - default='') - parser_migrate.add_argument( - "--min_size_mb", - help="min size of zfs image in megabytes (default: 512)", - type=int, - default=512) - parser_migrate.add_argument( - "--new_only", - help="if image already created, do nothing (default: False)", - default=False, - action="store_const", - const=True) - parser_migrate.add_argument( - "src", help="the current project home directory", type=str, nargs="+") - parser_migrate.set_defaults(func=migrate) - - def mount(args): - if not args.storage: - args.storage = os.environ['SALVUS_STORAGE'] - mount_project( - storage=args.storage, project_id=args.project_id, force=args.f) - - parser_mount = subparsers.add_parser( - 'mount', help='mount a project that is available in the storage pool') - parser_mount.add_argument( - "--storage", - help= - "the directory where project image directories are stored (default: $SALVUS_STORAGE enviro var)", - type=str, - default='') - - parser_mount.add_argument("project_id", help="the project id", type=str) - parser_mount.add_argument( - "-f", - help="force (default: False)", - default=False, - action="store_const", - const=True) - parser_mount.set_defaults(func=mount) - - def unmount(args): - unmount_project(project_id=args.project_id) - - parser_unmount = subparsers.add_parser( - 'umount', - help='unmount a project that is available in the storage pool') - parser_unmount.add_argument("project_id", help="the project id", type=str) - parser_unmount.set_defaults(func=unmount) - - def _info_json(args): - info_json(path=args.path) - - parser_migrate = subparsers.add_parser( - 'info_json', - help='query database, then write info.json file if there is none') - parser_migrate.add_argument( - "path", - help="path to a project home directory (old non-pooled)", - type=str, - nargs="+") - parser_migrate.set_defaults(func=_info_json) - - def _sync(args): - if not args.dest: - args.dest = ','.join(mount_target_volumes(args.volume)) - if not args.src: - args.src = ','.join(find_bricks(args.volume)) - - def main(): - while True: - try: - if args.watch: - sync_watch( - sources=args.src.split(','), - dests=args.dest.split(','), - min_sync_time=args.min_sync_time) - else: - for src in args.src.split(','): - for dest in args.dest.split(','): - sync(src=src, dest=dest) - except KeyboardInterrupt: - return - except Exception, mesg: - print mesg - if not args.daemon: - return - time.sleep(5) - - if args.daemon: - if not args.pidfile: - raise RuntimeError( - "in --daemon mode you *must* specify --pidfile") - import daemon - daemon.daemonize(args.pidfile) - main() - - parser_sync = subparsers.add_parser( - 'sync', - help= - 'Cross data center project sync: simply uses the local "cp" command and local mounts of the glusterfs, but provides massive speedups due to sparseness of image files' - ) - parser_sync.add_argument( - "--watch", - help= - "after running once, use inotify to watch for changes to the src filesystem and cp when they occur", - default=False, - action="store_const", - const=True) - parser_sync.add_argument( - "--min_sync_time", - help= - "never copy a file more frequently than this (default: 30 seconds)", - type=int, - default=30) - parser_sync.add_argument( - "--daemon", - help="daemon mode; will repeatedly sync", - dest="daemon", - default=False, - action="store_const", - const=True) - parser_sync.add_argument( - "--pidfile", - dest="pidfile", - type=str, - default='', - help="store pid in this file when daemonized") - parser_sync.add_argument( - "--dest", - help= - "comma separated list of destinations; if not given, all remote gluster volumes with name dc[n]-volume are mounted and targeted", - type=str, - default='') - parser_sync.add_argument( - "--src", - help= - "comma separated paths to bricks; if not given, local bricks for dc[n]-volume are used", - type=str, - default='') - parser_sync.add_argument( - "--volume", - help= - "if there are volumes dc0-projects, dc1-projects, dc2-projects, then pass option --volume=projects (default: 'projects')", - default='projects') - parser_sync.set_defaults(func=_sync) - - args = parser.parse_args() - - setup_log(loglevel=args.loglevel, logfile=args.logfile) - - args.func(args) - -else: - setup_log() From 687a8fd78778459e691f00286313a88c6b711c68 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 22 Jan 2025 03:04:41 +0000 Subject: [PATCH 013/281] nats: work in progress implementing proof of concept terminal to see how it goes - not done --- docs/nats/devlog.md | 70 ++++++++++++++----- src/packages/frontend/client/nats.ts | 8 ++- src/packages/pnpm-lock.yaml | 13 ++-- src/packages/project/nats/index.ts | 26 ++++--- src/packages/project/nats/terminal.ts | 99 +++++++++++++++++++++++++++ src/packages/project/package.json | 5 +- 6 files changed, 182 insertions(+), 39 deletions(-) create mode 100644 src/packages/project/nats/terminal.ts diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index cc60427f6f..696968240a 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -85,13 +85,13 @@ Maybe NATS totally kicks ass. - authentication: is there a way to too who the user who made the websocket connection is? - worry about this **later** \- obviously possible and not needed for a POC - let's try to make `write_text_file_to_project` also be possible via nats. -- OK, made some of api/v2 usable. Obviously this is really minimal POC. +- OK, made some of api/v2 usable. Obviously this is really minimal POC. ## [x] GOAL: do something involving the project -The most interesting use case for nats/jetsteam is timetravel collab editing, where this is all a VERY natural fit. +The most interesting use case for nats/jetsteam is timetravel collab editing, where this is all a VERY natural fit. -But for now, let's just do *something* at all. +But for now, let's just do _something_ at all. This worked - I did project exec with subject projects.{project_id}.api @@ -103,7 +103,7 @@ This worked - I did project exec with subject projects.{project_id}.api Creating a creds file that encodes a JWT that says what you can publish and subscribe to, then authenticating with that works. -- make it so user with account\_id can publish to hub.api.{account\_id} makes it so we know the account\_id automatically by virtue of what was published to. This works. +- make it so user with account_id can publish to hub.api.{account_id} makes it so we know the account_id automatically by virtue of what was published to. This works. ## [x] Goal: Solve Critical Auth Problems @@ -111,16 +111,16 @@ Now need to solve two problems: - [x] GOAL: set the creds for a browser client in a secure http cookie, so the browser can't directly access it -I finally figured this out after WASTING a lot of time with stupid AI misleading me and trying actively to get me to write very stupid insecure code as a lazy workaround. AI really is very, very dangerous... The trick was to read the docs repeatedly, increase logging a lot, and \-\- most imporantly \-\- read the relevant Go source code of NATS itself. The answer is to modify the JWT so that it explicitly has bearer set: `nsc edit user wstein --bearer` +I finally figured this out after WASTING a lot of time with stupid AI misleading me and trying actively to get me to write very stupid insecure code as a lazy workaround. AI really is very, very dangerous... The trick was to read the docs repeatedly, increase logging a lot, and \-\- most imporantly \-\- read the relevant Go source code of NATS itself. The answer is to modify the JWT so that it explicitly has bearer set: `nsc edit user wstein --bearer` -This makes it so the server doesn't check the signature of the JWT against the _user_ . Putting exactly the JWT token string in the cookie then works because "bearer" literally tells the backend server not to do the signature check. I think this is secure and the right approach because the server checks that the JWT is valid using the account and operator signatures. +This makes it so the server doesn't check the signature of the JWT against the _user_ . Putting exactly the JWT token string in the cookie then works because "bearer" literally tells the backend server not to do the signature check. I think this is secure and the right approach because the server checks that the JWT is valid using the account and operator signatures. -**WAIT!** Using signing keys [https://docs.nats.io/using\-nats/nats\-tools/nsc/signing\_keys](https://docs.nats.io/using-nats/nats-tools/nsc/signing_keys) \(and https://youtu.be/KmGtnFxHnVA?si=0uvLMBTJ5TUpem4O \) is VASTLY superior. There's just one JWT issued to each user, and we make a server\-side\-only JWT for their account that has everything. The user never has to reconnect or change their JWT. We can adjust the subject on the fly to account for running projects \(or collaboration changes\) at any time server side. Also the size limits go away, so we don't have to compress project\_id's \(probably\). +**WAIT!** Using signing keys [https://docs.nats.io/using\-nats/nats\-tools/nsc/signing_keys](https://docs.nats.io/using-nats/nats-tools/nsc/signing_keys) \(and https://youtu.be/KmGtnFxHnVA?si=0uvLMBTJ5TUpem4O \) is VASTLY superior. There's just one JWT issued to each user, and we make a server\-side\-only JWT for their account that has everything. The user never has to reconnect or change their JWT. We can adjust the subject on the fly to account for running projects \(or collaboration changes\) at any time server side. Also the size limits go away, so we don't have to compress project_id's \(probably\). ## Goal: Implement Auth Solution for Browsers - [x] automate creation of creds for browser clients, i.e., what we just did with the nsc tool manually -- +- --- @@ -130,9 +130,9 @@ What's the plan? Need to figure out how to do all the nsc stuff from javascript, storing results in the database? -- Question: how do we manage creating signing keys and users from nodejs? Answer: clear from many sources that we must use the nsc CLI tool via subprocess calls. Seems fine to me. -- [x] When a user signs in, we check for their JWT in the database. If it is there, set the cookie. If not, create the signing key and JWT for them, save in database, and set the cookie. -- [x] update nats\-server resolver state after modifying signing cookie's subjects configuration. +- Question: how do we manage creating signing keys and users from nodejs? Answer: clear from many sources that we must use the nsc CLI tool via subprocess calls. Seems fine to me. +- [x] When a user signs in, we check for their JWT in the database. If it is there, set the cookie. If not, create the signing key and JWT for them, save in database, and set the cookie. +- [x] update nats\-server resolver state after modifying signing cookie's subjects configuration. ``` nsc edit operator --account-jwt-server-url nats://localhost:4222 @@ -142,23 +142,55 @@ Now I can do `nsc push` and it just works. [x] TODO: when signing out, need to delete the jwt cookie or dangerous private info leaks... and also new info not set properly. -- [x] similar creds for projects, I.e., access to a project means you can publish to `projects.{project_id}.>` Also, projects should have access to something under hub. +- [x] similar creds for projects, I.e., access to a project means you can publish to `projects.{project_id}.>` Also, projects should have access to something under hub. -## Goal: Auth for Projects +## [x] Goal: Auth for Projects +Using an env variable I got a basic useful thing up and running. +--- +Some thoughts about project auth security: +- [ ] when collaborators on a project leave maybe we change JWT? Otherwise, in theory any user of a project can probably somehow get access to the project's JWT \(it's in memory at least\) and still act as the project. Changing JWT requires reconnect. This could be "for later", since even now we don't have this level of security! +- [ ] restarting project could change JWT. That's like the current project's secret token being changed. -Some thoughts about project auth security: -- [ ] when collaborators on a project leave maybe we change JWT? Otherwise, in theory any user of a project can probably somehow get access to the project's JWT \(it's in memory at least\) and still act as the project. Changing JWT requires reconnect. This could be "for later", since even now we don't have this level of security! -- [ ] restarting project could change JWT. That's like the current project's secret token being changed. +## [ ] Goal: nats-server automation of creation and configuration of system account, operator, etc. ---- +- This looks helpful: https://www.synadia.com/newsletter/nats-weekly-27/ +- NOT DONE YET + + +## [ ] Goal: Terminal! Something complicated involving the project which is NOT just request/response + +- Implementing terminals goes beyond request/response. +- It could also leverage jetstream if we want for state (?). +- Multiple connected client + + +Project/compute server sends terminal output to + + project.{project_id}.terminal.{sha1(path)} + +Anyone who can read project gets to see this. + +Browser sends terminal input to + + project.{project_id}.{group}.{account_id}.terminal.{sha1(path)} + +API calls: + + - to start terminal + - to get history (move to jetstream?) + +If I can get this to work, then collaborative editing and everything else is basically the same (just more details). + +Another thing to do for compute servers: + - use jetstream and KV to agree on *who* is running the terminal... + -## Goal: nats-server automation of creation and configuration of system account, operator, etc. -- This looks helpful: https://www.synadia.com/newsletter/nats-weekly-27/ + diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 73b0c6f343..d8a925cfd2 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -3,6 +3,7 @@ import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import type { WebappClient } from "./client"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { join } from "path"; +import { redux } from "../app-framework"; export class NatsClient { /*private*/ client: WebappClient; @@ -70,7 +71,12 @@ export class NatsClient { params?: object; }) => { const c = await this.getConnection(); - const subject = `project.${project_id}.owner.${this.client.account_id}.api`; + const group = redux.getProjectsStore().get_my_group(project_id); + if (!group) { + // todo...? + throw Error(`group not yet known for '${project_id}'`); + } + const subject = `project.${project_id}.${group}.${this.client.account_id}.api`; const resp = await c.request( subject, this.jc.encode({ diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 22650652f5..ecb0fd6ca6 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1292,6 +1292,9 @@ importers: nats: specifier: ^2.29.1 version: 2.29.1 + node-pty: + specifier: ^1.0.0 + version: 1.0.0 pidusage: specifier: ^1.2.0 version: 1.2.0 @@ -9400,9 +9403,6 @@ packages: nan@2.17.0: resolution: {integrity: sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==} - nan@2.19.0: - resolution: {integrity: sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==} - nan@2.20.0: resolution: {integrity: sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==} @@ -16972,7 +16972,7 @@ snapshots: canvas@2.11.2(encoding@0.1.13): dependencies: '@mapbox/node-pre-gyp': 1.0.11(encoding@0.1.13) - nan: 2.19.0 + nan: 2.20.0 simple-get: 3.1.1 transitivePeerDependencies: - encoding @@ -21582,9 +21582,6 @@ snapshots: nan@2.17.0: {} - nan@2.19.0: - optional: true - nan@2.20.0: {} nanoid@3.3.8: {} @@ -21798,7 +21795,7 @@ snapshots: node-pty@1.0.0: dependencies: - nan: 2.17.0 + nan: 2.20.0 node-releases@2.0.18: {} diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts index c3793f1b9a..8eddfb9d9a 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/nats/index.ts @@ -20,6 +20,8 @@ How to do development (so in a dev project doing cc-in-cc dev). import { getLogger } from "@cocalc/project/logger"; import { connect, JSONCodec, jwtAuthenticator } from "nats"; import { project_id } from "@cocalc/project/data"; +import { handleExecShellCode } from "@cocalc/project/exec_shell_code"; +import { realpath } from "@cocalc/project/browser-websocket/realpath"; const logger = getLogger("server:nats"); @@ -42,29 +44,33 @@ export async function initAPI(nc) { logger.debug(`initAPI -- NATS project subject '${subject}'`); const sub = nc.subscribe(subject); for await (const mesg of sub) { - handleRequest(mesg); + handleRequest(mesg, nc); } } -async function handleRequest(mesg) { +async function handleRequest(mesg, nc) { const segments = mesg.subject.split("."); const group = segments[2]; // 'owner', 'collaborator', etc. const account_id = segments[3]; - const type = segments[4]; // e.g., 'api', etc. + const type = segments[4]; // e.g., 'api', etc. (?) if (type == "api") { - await handleApiRequest({ mesg, group, account_id }); + await handleApiRequest({ mesg, group, account_id, nc }); } else { logger.debug(`unknown request type '${type}'`); } } -async function handleApiRequest({ mesg, group, account_id }) { +async function handleApiRequest({ mesg, group, account_id, nc }) { const request = jc.decode(mesg.data) ?? {}; let resp; try { const { endpoint, params } = request as any; logger.debug("handling project request:", { endpoint, group, account_id }); - resp = await getResponse({ endpoint, params }); + if (endpoint == "write-to-terminal") { + // no response needed + return writeToTerminal(params); + } + resp = await getResponse({ endpoint, params, nc }); } catch (err) { resp = { error: `${err}` }; } @@ -72,10 +78,8 @@ async function handleApiRequest({ mesg, group, account_id }) { mesg.respond(jc.encode(resp)); } -import { handleExecShellCode } from "@cocalc/project/exec_shell_code"; -import { realpath } from "@cocalc/project/browser-websocket/realpath"; - -async function getResponse({ endpoint, params }) { +import { createTerminal, writeToTerminal } from "./terminal"; +async function getResponse({ endpoint, params, nc }) { switch (endpoint) { case "ping": return { pong: Date.now() }; @@ -83,6 +87,8 @@ async function getResponse({ endpoint, params }) { return realpath(params.path); case "exec": return await handleExecShellCode(params); + case "create-terminal": + return await createTerminal({ params, nc }); default: throw Error(`unknown endpoint '${endpoint}'`); } diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts new file mode 100644 index 0000000000..9c6bf9122a --- /dev/null +++ b/src/packages/project/nats/terminal.ts @@ -0,0 +1,99 @@ +/* +Quick very simple terminal proof of concept for testing NATS +*/ + +import { spawn } from "node-pty"; +import { envForSpawn } from "@cocalc/backend/misc"; +import { path_split } from "@cocalc/util/misc"; +import { getCWD } from "@cocalc/terminal/lib/util"; +import { console_init_filename } from "@cocalc/util/misc"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { project_id } from "@cocalc/project/data"; +import { sha1 } from "@cocalc/backend/sha1"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; + +const DEFAULT_COMMAND = "/bin/bash"; + +const sessions: { [name: string]: Session } = {}; + +export const createTerminal = reuseInFlight( + async ({ params, nc }: { params; nc }) => { + if (params == null) { + throw Error("params must be specified"); + } + const { path, options } = params; + if (!path) { + throw Error("path must be specified"); + } + if (sessions[path] == null) { + sessions[path] = new Session({ path, options, nc }); + await sessions[path].init(); + } + return sessions[path].subject; + }, + { + createKey: (args) => { + return args[0]?.params?.path ?? ""; + }, + }, +); + +export function writeToTerminal({ data, path }: { data; path }) { + sessions[path]?.write(data); +} + +class Session { + private nc; + private path: string; + private options?; + private pty?; + private size?: { rows: number; cols: number }; + // the subject where we publish our output + public subject: string; + + constructor({ path, options, nc }) { + this.nc = nc; + this.path = path; + this.options = options; + this.subject = `project.${project_id}.terminal.${sha1(path)}`; + } + + write = (data) => { + if (this.pty == null) { + return; + } + this.pty?.(data); + }; + + init = async () => { + const { head, tail } = path_split(this.path); + const env = { + COCALC_TERMINAL_FILENAME: tail, + ...envForSpawn(), + ...this.options?.env, + TMUX: undefined, // ensure not set + }; + const command = this.options?.command ?? DEFAULT_COMMAND; + const args = this.options.args ?? []; + const initFilename: string = console_init_filename(this.path); + if (await exists(initFilename)) { + args.push("--init-file"); + args.push(path_split(initFilename).tail); + } + const cwd = getCWD(head, this.options?.cwd); + this.pty = spawn(command, args, { + cwd, + env, + rows: this.size?.rows, + cols: this.size?.cols, + }); + + this.pty.onData((data) => { + this.nc.publish(this.subject, data); + }); + this.pty.onExit(() => { + // todo + console.log("exit"); + }); + }; +} diff --git a/src/packages/project/package.json b/src/packages/project/package.json index 0d08e2d19b..46d13ecb22 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -54,6 +54,7 @@ "lodash": "^4.17.21", "lru-cache": "^7.18.3", "nats": "^2.29.1", + "node-pty": "^1.0.0", "pidusage": "^1.2.0", "prettier": "^3.0.2", "primus": "^8.0.9", @@ -84,7 +85,9 @@ "clean": "rm -rf dist" }, "author": "SageMath, Inc.", - "contributors": ["William Stein "], + "contributors": [ + "William Stein " + ], "license": "SEE LICENSE.md", "bugs": { "url": "https://github.com/sagemathinc/cocalc/issues" From 59da3c9260bb00a596bb20d44bc0511118376813 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 22 Jan 2025 04:39:12 +0000 Subject: [PATCH 014/281] nats: working terminal communication (very basic but works) --- src/packages/frontend/client/nats.ts | 11 +++++- src/packages/project/nats/index.ts | 30 +++++++-------- src/packages/project/nats/terminal.ts | 55 ++++++++++++++++++++++----- src/packages/server/nats/auth.ts | 3 +- 4 files changed, 73 insertions(+), 26 deletions(-) diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index d8a925cfd2..c681316431 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -76,7 +76,7 @@ export class NatsClient { // todo...? throw Error(`group not yet known for '${project_id}'`); } - const subject = `project.${project_id}.${group}.${this.client.account_id}.api`; + const subject = `project.${project_id}.api.${group}.${this.client.account_id}`; const resp = await c.request( subject, this.jc.encode({ @@ -86,4 +86,13 @@ export class NatsClient { ); return this.jc.decode(resp.data); }; + + // for debugging -- listen to and display all messages on a subject + subscribe = async (subject: string) => { + const nc = await this.getConnection(); + const sub = nc.subscribe(subject); + for await (const mesg of sub) { + console.log(this.jc.decode(mesg.data)); + } + }; } diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts index 8eddfb9d9a..d6e4701dd1 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/nats/index.ts @@ -40,7 +40,7 @@ export default async function initNatsServer() { const jc = JSONCodec(); export async function initAPI(nc) { - const subject = `project.${project_id}.>`; + const subject = `project.${project_id}.api.>`; logger.debug(`initAPI -- NATS project subject '${subject}'`); const sub = nc.subscribe(subject); for await (const mesg of sub) { @@ -50,14 +50,9 @@ export async function initAPI(nc) { async function handleRequest(mesg, nc) { const segments = mesg.subject.split("."); - const group = segments[2]; // 'owner', 'collaborator', etc. - const account_id = segments[3]; - const type = segments[4]; // e.g., 'api', etc. (?) - if (type == "api") { - await handleApiRequest({ mesg, group, account_id, nc }); - } else { - logger.debug(`unknown request type '${type}'`); - } + const group = segments[3]; // 'owner', 'collaborator', etc. + const account_id = segments[4]; + await handleApiRequest({ mesg, group, account_id, nc }); } async function handleApiRequest({ mesg, group, account_id, nc }) { @@ -65,11 +60,12 @@ async function handleApiRequest({ mesg, group, account_id, nc }) { let resp; try { const { endpoint, params } = request as any; - logger.debug("handling project request:", { endpoint, group, account_id }); - if (endpoint == "write-to-terminal") { - // no response needed - return writeToTerminal(params); - } + logger.debug("handling project request:", { + endpoint, + params, + group, + account_id, + }); resp = await getResponse({ endpoint, params, nc }); } catch (err) { resp = { error: `${err}` }; @@ -78,7 +74,7 @@ async function handleApiRequest({ mesg, group, account_id, nc }) { mesg.respond(jc.encode(resp)); } -import { createTerminal, writeToTerminal } from "./terminal"; +import { createTerminal, restartTerminal, writeToTerminal } from "./terminal"; async function getResponse({ endpoint, params, nc }) { switch (endpoint) { case "ping": @@ -89,6 +85,10 @@ async function getResponse({ endpoint, params, nc }) { return await handleExecShellCode(params); case "create-terminal": return await createTerminal({ params, nc }); + case "restart-terminal": + return await restartTerminal(params); + case "write-to-terminal": + return writeToTerminal(params); default: throw Error(`unknown endpoint '${endpoint}'`); } diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index 9c6bf9122a..6d8400c3d2 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -5,14 +5,15 @@ Quick very simple terminal proof of concept for testing NATS import { spawn } from "node-pty"; import { envForSpawn } from "@cocalc/backend/misc"; import { path_split } from "@cocalc/util/misc"; -import { getCWD } from "@cocalc/terminal/lib/util"; import { console_init_filename } from "@cocalc/util/misc"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { project_id } from "@cocalc/project/data"; import { sha1 } from "@cocalc/backend/sha1"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { JSONCodec } from "nats"; const DEFAULT_COMMAND = "/bin/bash"; +const jc = JSONCodec(); const sessions: { [name: string]: Session } = {}; @@ -29,7 +30,7 @@ export const createTerminal = reuseInFlight( sessions[path] = new Session({ path, options, nc }); await sessions[path].init(); } - return sessions[path].subject; + return { subject: sessions[path].subject }; }, { createKey: (args) => { @@ -39,7 +40,21 @@ export const createTerminal = reuseInFlight( ); export function writeToTerminal({ data, path }: { data; path }) { - sessions[path]?.write(data); + const terminal = sessions[path]; + if (terminal == null) { + throw Error(`no terminal session '${path}'`); + } + terminal.write(data); + return { success: true }; +} + +export async function restartTerminal({ path }: { path }) { + const terminal = sessions[path]; + if (terminal == null) { + throw Error(`no terminal session '${path}'`); + } + await terminal.restart(); + return { success: true }; } class Session { @@ -59,10 +74,17 @@ class Session { } write = (data) => { + console.log("write", { data }); if (this.pty == null) { return; } - this.pty?.(data); + this.pty.write(data); + }; + + restart = async () => { + this.pty?.destroy(); + delete this.pty; + await this.init(); }; init = async () => { @@ -74,7 +96,7 @@ class Session { TMUX: undefined, // ensure not set }; const command = this.options?.command ?? DEFAULT_COMMAND; - const args = this.options.args ?? []; + const args = this.options?.args ?? []; const initFilename: string = console_init_filename(this.path); if (await exists(initFilename)) { args.push("--init-file"); @@ -89,11 +111,26 @@ class Session { }); this.pty.onData((data) => { - this.nc.publish(this.subject, data); + // console.log("onData", { data }); + this.nc.publish(this.subject, jc.encode({ data })); }); - this.pty.onExit(() => { - // todo - console.log("exit"); + this.pty.onExit((status) => { + this.nc.publish(this.subject, jc.encode({ ...status, exit: true })); }); }; } + +function getCWD(pathHead, cwd?): string { + // working dir can be set explicitly, and either be an empty string or $HOME + if (cwd != null) { + const HOME = process.env.HOME ?? "/home/user"; + if (cwd === "") { + return HOME; + } else if (cwd.startsWith("$HOME")) { + return cwd.replace("$HOME", HOME); + } else { + return cwd; + } + } + return pathHead; +} diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index fcfcc795d5..9bbbef381f 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -106,7 +106,8 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { const query = `SELECT project_id, users#>>'{${userId},group}' AS group FROM projects WHERE state#>>'{state}'='running' AND users ? '${userId}' ORDER BY project_id`; const { rows } = await pool.query(query); for (const { project_id, group } of rows) { - goalPub.add(`project.${project_id}.${group}.${userId}.>`); + goalPub.add(`project.${project_id}.api.${group}.${userId}`); + goalSub.add(`project.${project_id}.>`); } // TODO: there will be other subjects // TODO: something similar for projects, e.g., they can publish to a channel that browser clients From cc955ea7fa45260f32859ccef333472ed6e7c486 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 22 Jan 2025 16:07:13 +0000 Subject: [PATCH 015/281] nats: terminal proof of concept --- docs/nats/devlog.md | 14 ++++- .../terminal-editor/connected-terminal.ts | 23 +++++++++ .../nats-terminal-connection.ts | 51 +++++++++++++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 696968240a..655aa4704c 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -163,7 +163,7 @@ Some thoughts about project auth security: - NOT DONE YET -## [ ] Goal: Terminal! Something complicated involving the project which is NOT just request/response +## [x] Goal: Terminal! Something complicated involving the project which is NOT just request/response - Implementing terminals goes beyond request/response. - It could also leverage jetstream if we want for state (?). @@ -187,9 +187,21 @@ API calls: If I can get this to work, then collaborative editing and everything else is basically the same (just more details). +## [ ] Goal: Terminal! #now + +Make it so an actual terminal works, i.e., UI integration. + +## [ ] Goal: Terminal JetStream state + +Use Jetstream to store messages from terminal, so user can reconnect without loss. + +## [ ] Goal: Terminal and **compute server** + Another thing to do for compute servers: - use jetstream and KV to agree on *who* is running the terminal... +This is critical to see how easily we can support compute servers using nats + jetstream. + diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 574d3adb63..fe2ea65387 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -40,6 +40,7 @@ import { ConnectedTerminalInterface } from "./connected-terminal-interface"; import { open_init_file } from "./init-file"; import { setTheme } from "./themes"; import { modalParams } from "@cocalc/frontend/compute/select-server-for-file"; +import { NatsTerminalConnection } from "./nats-terminal-connection"; declare const $: any; @@ -271,7 +272,29 @@ export class Terminal { this.terminal_settings = settings; } + connectNats = async () => { + try { + this.ignore_terminal_data = false; // todo + this.set_connection_status("connecting"); + const conn = new NatsTerminalConnection({ + path: this.term_path, + project_id: this.project_id, + }); + this.conn = conn as any; + conn?.on("close", this.connect); + conn?.on("data", this._handle_data_from_project); + await conn?.init(); + this.set_connection_status("connected"); + } catch (err) { + this.set_connection_status("disconnected"); + throw err; + } + }; + async connect(): Promise { + if (this.path == "nats.term") { + return await this.connectNats(); + } this.assert_not_closed(); this.last_geom = undefined; diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts new file mode 100644 index 0000000000..905848cece --- /dev/null +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -0,0 +1,51 @@ +import { webapp_client } from "@cocalc/frontend/webapp-client"; +import { EventEmitter } from "events"; +import { JSONCodec } from "nats.ws"; + +export class NatsTerminalConnection extends EventEmitter { + private project_id: string; + private path: string; + + constructor({ project_id, path }) { + super(); + this.project_id = project_id; + this.path = path; + } + + write = async (data) => { + if (typeof data != "string") { + console.log("write -- todo:", data); + // TODO: not yet implemented, e.g., {cmd: 'size', rows: 18, cols: 180} + return; + } + await webapp_client.nats_client.project({ + project_id: this.project_id, + endpoint: "write-to-terminal", + params: { path: this.path, data }, + }); + }; + + end = () => { + // todo + }; + + init = async () => { + const jc = JSONCodec(); + const { subject } = (await webapp_client.nats_client.project({ + project_id: this.project_id, + endpoint: "create-terminal", + params: { path: this.path }, + })) as any; + const nc = await webapp_client.nats_client.getConnection(); + const sub = nc.subscribe(subject); + for await (const mesg of sub) { + const { exit, data } = jc.decode(mesg.data) as any; + if (exit) { + this.emit("close"); + break; + } else if (data != null) { + this.emit("data", data); + } + } + }; +} From e884b4fd40820d158b7361a24b3c6b6c4f7311d4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 22 Jan 2025 18:42:55 +0000 Subject: [PATCH 016/281] nats: sys --> cocalc account --- docs/nats/devlog.md | 18 ++++++++++++++++-- src/packages/server/nats/auth.ts | 27 ++++++++++++++++++++++----- src/packages/server/nats/index.ts | 4 ++-- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 655aa4704c..bdbeec2c8d 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -187,13 +187,27 @@ API calls: If I can get this to work, then collaborative editing and everything else is basically the same (just more details). -## [ ] Goal: Terminal! #now +## [x] Goal: Terminal! #now Make it so an actual terminal works, i.e., UI integration. ## [ ] Goal: Terminal JetStream state -Use Jetstream to store messages from terminal, so user can reconnect without loss. +Use Jetstream to store messages from terminal, so user can reconnect without loss. !? This is very interesting... + +First problem -- we used the system account SYS for all our users; however, +SYS can't use jetstreams, as explained here https://github.com/nats-io/nats-server/discussions/6033 + +Let's redo *everything* with a new account called "cocalc". + +```sh +~/nats$ nsc create account --name=cocalc +[ OK ] generated and stored account key "AD4G6R62BDDQUSCJVLZNA7ES7R3A6DWXLYUWGZV74EJ2S6VBC7DQVM3I" +[ OK ] added account "cocalc" +~/nats$ nats context save admin --creds=/projects/3fa218e5-7196-4020-8b30-e2127847cc4f/.local/share/nats/nsc/keys/creds/MyOperator/cocalc/admin.creds +~/nats$ nsc edit account cocalc --js-enable 1 +~/nats$ nsc push -a cocalc +``` ## [ ] Goal: Terminal and **compute server** diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 9bbbef381f..a5d60fea89 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -11,6 +11,10 @@ Points that took me a while to figure out: TODO: worry about expiration - There is no supported way to do user management except calling the nsc command line tool. That's fine. + + +DOCS: + - https://nats-io.github.io/nsc/ */ import { executeCode } from "@cocalc/backend/execute-code"; @@ -20,13 +24,22 @@ import getLogger from "@cocalc/backend/logger"; import { throttle } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +// TODO: move this to server settings +const NATS_ACCOUNT = "cocalc"; + const logger = getLogger("server:nats:auth"); -export async function nsc(args: string[]) { +export async function nsc( + args: string[], + { noAccount }: { noAccount?: boolean } = {}, +) { // todo: for production we have to put some authentication // options, e.g., taken from the database. Skip that for now. // console.log(`nsc ${args.join(" ")}`); - return await executeCode({ command: "nsc", args }); + return await executeCode({ + command: "nsc", + args: noAccount ? args : [...args, "-a", NATS_ACCOUNT], + }); } // TODO: consider making the names shorter strings using https://www.npmjs.com/package/short-uuid @@ -229,7 +242,7 @@ export async function getScopedSigningKey(natsUser: string) { export const pushToServer = throttle( reuseInFlight(async () => { try { - await nsc(["push", "-a", "SYS"]); + await nsc(["push"]); } catch (err) { // TODO: adminNotification? This could be very serious. logger.debug("push configuration to nats server failed", err); @@ -240,8 +253,12 @@ export const pushToServer = throttle( ); export async function createNatsUser(cocalcUser: CoCalcUser) { - await nsc(["pull", "-A"]); - const { stderr } = await nsc(["edit", "account", "--sk", "generate"]); + await nsc(["pull", "-A"], { + noAccount: true, + }); + const { stderr } = await nsc(["edit", "account", "--sk", "generate"], { + noAccount: true, + }); const key = stderr.trim().split('"')[1]; const name = getNatsUserName(cocalcUser); // bearer is critical so that the signing key can be used in the browser without diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts index b16fa3b7ca..5d7cd32508 100644 --- a/src/packages/server/nats/index.ts +++ b/src/packages/server/nats/index.ts @@ -5,7 +5,7 @@ import { initAPI } from "./api"; const logger = getLogger("server:nats"); const creds = `-----BEGIN NATS USER JWT----- -eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJFNUhFT1g3VFJETVNCVzdWRUNTVkRDRVlZVkRON0lZNUMyVzZNWEw2RVRVQVJSNDVZTkhBIiwiaWF0IjoxNzM3NDkxNTMwLCJpc3MiOiJBQjJLVEVGUFIyTzc2UE9aVVBZRVFTS1RaQVg2R0lOQVZUNkpXU0g2UUI3TENNNFhIRlRITVgyTCIsIm5hbWUiOiJodWIiLCJzdWIiOiJVQUhTQ1hVUVEzSFVVRlFIV0xUN0tYVDRXSU1ZSkdSQ1VLUUROWEZQRURCNU1WWkNMNkJKTldWVSIsIm5hdHMiOnsicHViIjp7ImFsbG93IjpbIl9JTkJPWC5cdTAwM2UiXX0sInN1YiI6eyJhbGxvdyI6WyJodWIuXHUwMDNlIl19LCJzdWJzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJ0eXBlIjoidXNlciIsInZlcnNpb24iOjJ9fQ.yaXnOnTFqJQvTwkdWpRS9nQSSZJrUKJRqJYcqUj1ymz_eDdEZ-UdKrFCIFT7GSkbhIRlt5E6GCAeYZx2X9brCg +eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJVTDJaWEdFWFFKTzVRNjdLU1hKNDdERFpKSFE3QUFWMjdHWUtBN1ZJVjVaT01DQU1SN1hBIiwiaWF0IjoxNzM3NTY3OTQwLCJpc3MiOiJBRDRHNlI2MkJERFFVU0NKVkxaTkE3RVM3UjNBNkRXWExZVVdHWlY3NEVKMlM2VkJDN0RRVk0zSSIsIm5hbWUiOiJhZG1pbiIsInN1YiI6IlVBV1hZVUpYSEFXQzNPSFFURE1SQVBSWVpNNFQ0RkZDRk1TTVFLNDVCWU1SS0ZSRE5RTjQ0Vk1SIiwibmF0cyI6eyJwdWIiOnsiYWxsb3ciOlsiXHUwMDNlIl19LCJzdWIiOnsiYWxsb3ciOlsiXHUwMDNlIl19LCJzdWJzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJ0eXBlIjoidXNlciIsInZlcnNpb24iOjJ9fQ.Pv9-T3P7cO1VSFiNocGA0vCGvwQ-UaX3b7OzwMIHdn5hGs4kUv4eLE-Er_6dxrZiPu6PJjBYB7eD2hyb-gxSCQ ------END NATS USER JWT------ ************************* IMPORTANT ************************* @@ -13,7 +13,7 @@ NKEY Seed printed below can be used to sign and prove identity. NKEYs are sensitive and should be treated as secrets. -----BEGIN USER NKEY SEED----- -SUAOMBSB4Z6XVTXWQCVZPG2OWM6C36UTP6O47ILTW3LC75HW5U2QCE3C5U +SUAMW6S2OXSKL2ETX5GJE3NDLWGXZFZ4JAP5WHBCK43RMFDPJCCJLPWC5Y ------END USER NKEY SEED------ ************************************************************* From 5fdbb7926aafa5958b7a517d4a35fa38d25b62ec Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 22 Jan 2025 22:27:00 +0000 Subject: [PATCH 017/281] nats: work in progress using jetstream for terminals --- docs/nats/devlog.md | 36 ++-- src/packages/frontend/client/nats.ts | 7 + .../nats-terminal-connection.ts | 70 ++++++-- src/packages/frontend/package.json | 1 + src/packages/pnpm-lock.yaml | 162 +++++++++--------- src/packages/project/nats/index.ts | 2 +- src/packages/project/nats/terminal.ts | 21 ++- src/packages/server/nats/auth.ts | 3 + 8 files changed, 185 insertions(+), 117 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index bdbeec2c8d..ea0002d7a0 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -155,23 +155,19 @@ Some thoughts about project auth security: - [ ] when collaborators on a project leave maybe we change JWT? Otherwise, in theory any user of a project can probably somehow get access to the project's JWT \(it's in memory at least\) and still act as the project. Changing JWT requires reconnect. This could be "for later", since even now we don't have this level of security! - [ ] restarting project could change JWT. That's like the current project's secret token being changed. - - ## [ ] Goal: nats-server automation of creation and configuration of system account, operator, etc. - This looks helpful: https://www.synadia.com/newsletter/nats-weekly-27/ - NOT DONE YET - ## [x] Goal: Terminal! Something complicated involving the project which is NOT just request/response - Implementing terminals goes beyond request/response. - It could also leverage jetstream if we want for state (?). - Multiple connected client - Project/compute server sends terminal output to - + project.{project_id}.terminal.{sha1(path)} Anyone who can read project gets to see this. @@ -179,7 +175,7 @@ Anyone who can read project gets to see this. Browser sends terminal input to project.{project_id}.{group}.{account_id}.terminal.{sha1(path)} - + API calls: - to start terminal @@ -191,7 +187,7 @@ If I can get this to work, then collaborative editing and everything else is bas Make it so an actual terminal works, i.e., UI integration. -## [ ] Goal: Terminal JetStream state +## [x] Goal: Terminal JetStream state Use Jetstream to store messages from terminal, so user can reconnect without loss. !? This is very interesting... @@ -209,14 +205,30 @@ Let's redo *everything* with a new account called "cocalc". ~/nats$ nsc push -a cocalc ``` +```js +// making the stream for ALL terminal activity +await jsm.streams.add({ name: 'project-81e0c408-ac65-4114-bad5-5f4b6539bd0e-terminal', subjects: ['project.81e0c408-ac65-4114-bad5-5f4b6539bd0e.terminal.>'] }); + +// making a consumer for just one subject (e.g., one terminal frame) +z = await jsm.consumers.add('project-81e0c408-ac65-4114-bad5-5f4b6539bd0e-terminal',{name:'9149af7632942a94ea13877188153bd8bf2ace57',filter:['project.81e0c408-ac65-4114-bad5-5f4b6539bd0e.terminal.9149af7632942a94ea13877188153bd8bf2ace57']}) +c = await js.consumers.get('project-81e0c408-ac65-4114-bad5-5f4b6539bd0e-terminal', '9149af7632942a94ea13877188153bd8bf2ace57') +for await (const m of await c.consume()) { console.log(cc.client.nats_client.jc.decode(m.data))} +``` + +NOTE!!! The above consumer is ephemeral -- it disappears if we don't grab it via c within a few seconds!!!! https://docs.nats.io/using-nats/developer/develop_jetstream/consumers + +## [ ] Goal: Jetstream permissions + +- [ ] who sets up the stream for capturing terminal outputs and when? +- [ ] what are the permissions for jetstream usage and access? +- [ ] deleting old data? +- [ ] handle the other messages like resize + ## [ ] Goal: Terminal and **compute server** Another thing to do for compute servers: - - use jetstream and KV to agree on *who* is running the terminal... - -This is critical to see how easily we can support compute servers using nats + jetstream. - +- use jetstream and KV to agree on _who_ is running the terminal? +This is critical to see how easily we can support compute servers using nats + jetstream. - diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index c681316431..44a43a8406 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -4,6 +4,7 @@ import type { WebappClient } from "./client"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { join } from "path"; import { redux } from "../app-framework"; +import * as jetstream from "@nats-io/jetstream"; export class NatsClient { /*private*/ client: WebappClient; @@ -12,6 +13,7 @@ export class NatsClient { private nc?: Awaited>; // obviously just for learning: public nats = nats; + public jetstream = jetstream; constructor(client: WebappClient) { this.client = client; @@ -95,4 +97,9 @@ export class NatsClient { console.log(this.jc.decode(mesg.data)); } }; + + consumer = async (stream: string) => { + const js = jetstream.jetstream(await await this.getConnection()); + return await js.consumers.get(stream); + }; } diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 905848cece..1a0b11d651 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -1,49 +1,89 @@ import { webapp_client } from "@cocalc/frontend/webapp-client"; import { EventEmitter } from "events"; import { JSONCodec } from "nats.ws"; +import sha1 from "sha1"; export class NatsTerminalConnection extends EventEmitter { private project_id: string; private path: string; + private subject: string; + private state: null | "running" | "off"; constructor({ project_id, path }) { super(); this.project_id = project_id; this.path = path; + // move to util so guaranteed in sync with project + this.subject = `project.${project_id}.terminal.${sha1(path)}`; } write = async (data) => { + if (this.state != "running") { + await this.start(); + } if (typeof data != "string") { - console.log("write -- todo:", data); + //console.log("write -- todo:", data); // TODO: not yet implemented, e.g., {cmd: 'size', rows: 18, cols: 180} return; } - await webapp_client.nats_client.project({ - project_id: this.project_id, - endpoint: "write-to-terminal", - params: { path: this.path, data }, - }); + const write = async () => { + await webapp_client.nats_client.project({ + project_id: this.project_id, + endpoint: "write-to-terminal", + params: { path: this.path, data }, + }); + }; + try { + await write(); + } catch (_) { + await this.start(); + await write(); + } }; end = () => { // todo }; - init = async () => { - const jc = JSONCodec(); - const { subject } = (await webapp_client.nats_client.project({ + private start = async () => { + // ensure running: + await webapp_client.nats_client.project({ project_id: this.project_id, endpoint: "create-terminal", params: { path: this.path }, - })) as any; - const nc = await webapp_client.nats_client.getConnection(); - const sub = nc.subscribe(subject); - for await (const mesg of sub) { + }); + }; + + private getConsumer = async () => { + // TODO: idempotent, but move to project + const { nats_client } = webapp_client; + const stream = `project-${this.project_id}-terminal`; + const nc = await nats_client.getConnection(); + const js = nats_client.jetstream.jetstream(nc); + // consumer doesn't exist, so setup everything. + const jsm = await nats_client.jetstream.jetstreamManager(nc); + await jsm.streams.add({ + name: stream, + subjects: [`project.${this.project_id}.terminal.>`], + compression: "s2", + }); + // making an ephemeral consumer for just one subject (e.g., this terminal frame) + const { name } = await jsm.consumers.add(stream, { + filter_subject: this.subject, + }); + return await js.consumers.get(stream, name); + }; + + init = async () => { + const jc = JSONCodec(); + const consumer = await this.getConsumer(); + await this.start(); + for await (const mesg of await consumer.consume()) { const { exit, data } = jc.decode(mesg.data) as any; if (exit) { - this.emit("close"); - break; + this.state = "off"; } else if (data != null) { + this.state = "running"; this.emit("data", data); } } diff --git a/src/packages/frontend/package.json b/src/packages/frontend/package.json index 540525ab45..e211818272 100644 --- a/src/packages/frontend/package.json +++ b/src/packages/frontend/package.json @@ -57,6 +57,7 @@ "@jupyter-widgets/output": "^4.1.0", "@lumino/widgets": "^1.31.1", "@microlink/react-json-view": "^1.23.3", + "@nats-io/jetstream": "3.0.0-36", "@orama/orama": "3.0.0-rc-3", "@react-hook/mouse-position": "^4.1.3", "@rinsuki/lz4-ts": "^1.0.1", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index ecb0fd6ca6..deb3c2f072 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -191,7 +191,7 @@ importers: version: 8.7.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0(supports-color@9.4.0) immutable: specifier: ^4.3.0 version: 4.3.7 @@ -301,6 +301,9 @@ importers: '@microlink/react-json-view': specifier: ^1.23.3 version: 1.23.3(@types/react@18.3.10)(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@nats-io/jetstream': + specifier: 3.0.0-36 + version: 3.0.0-36 '@orama/orama': specifier: 3.0.0-rc-3 version: 3.0.0-rc-3 @@ -396,7 +399,7 @@ importers: version: 1.11.13 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0(supports-color@9.4.0) direction: specifier: ^1.0.4 version: 1.0.4 @@ -799,7 +802,7 @@ importers: version: 2.8.5 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0(supports-color@9.4.0) escape-html: specifier: ^1.0.3 version: 1.0.3 @@ -977,7 +980,7 @@ importers: version: 8.7.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0(supports-color@9.4.0) enchannel-zmq-backend: specifier: ^9.1.23 version: 9.1.23(rxjs@7.8.1) @@ -1252,7 +1255,7 @@ importers: version: 3.0.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0(supports-color@9.4.0) diskusage: specifier: ^1.1.3 version: 1.2.0 @@ -1837,7 +1840,7 @@ importers: version: 3.0.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0(supports-color@9.4.0) events: specifier: 3.3.0 version: 3.3.0 @@ -1889,7 +1892,7 @@ importers: version: 1.0.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0(supports-color@9.4.0) primus: specifier: ^8.0.9 version: 8.0.9 @@ -1969,7 +1972,7 @@ importers: version: 3.0.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0(supports-color@9.4.0) lodash: specifier: ^4.17.21 version: 4.17.21 @@ -2015,7 +2018,7 @@ importers: version: 1.11.13 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0(supports-color@9.4.0) decimal.js-light: specifier: ^2.5.1 version: 2.5.1 @@ -3848,6 +3851,20 @@ packages: '@module-federation/webpack-bundler-runtime@0.5.1': resolution: {integrity: sha512-mMhRFH0k2VjwHt3Jol9JkUsmI/4XlrAoBG3E0o7HoyoPYv1UFOWyqAflfANcUPgbYpvqmyLzDcO+3IT36LXnrA==} + '@nats-io/jetstream@3.0.0-36': + resolution: {integrity: sha512-95+ftM+lXcUEwCl3ctzpJZCXiBUwRuSUUWmd4X8jJlNVZc/LUHH+3cxX+nC42sLHD+9zvIaMKPsyI0Ltbp+BSg==} + + '@nats-io/nats-core@3.0.0-49': + resolution: {integrity: sha512-Xe7LjCdhtL4pXk2czwUE8Y1elTy/zo3ZzpoIwOO+/uJPughEsSxCpqygPrDqWcOG2uWVB9G1wxjg8r0Y9StovQ==} + + '@nats-io/nkeys@2.0.2': + resolution: {integrity: sha512-0JTyVl9P+UJyjUBDWP9589TuUKXJQ8tDkVRgi02X/MMzW997+4FykirvZEkIe6ZAhiLIBN+NpN8ULMMt6mDrbA==} + engines: {node: '>=18.0.0'} + + '@nats-io/nuid@2.0.3': + resolution: {integrity: sha512-TpA3HEBna/qMVudy+3HZr5M3mo/L1JPofpVT4t0HkFGkz2Cn9wrlrQC8tvR8Md5Oa9//GtGG26eN0qEWF5Vqew==} + engines: {node: '>= 18.x'} + '@nestjs/axios@3.0.3': resolution: {integrity: sha512-h6TCn3yJwD6OKqqqfmtRS5Zo4E46Ip2n+gK1sqwzNBC+qxQ9xpCu+ODVRFur6V3alHSCSBxb3nNtt73VEdluyA==} peerDependencies: @@ -12945,10 +12962,10 @@ snapshots: '@babel/helpers': 7.25.6 '@babel/parser': 7.25.6 '@babel/template': 7.25.0 - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.25.6(supports-color@9.4.0) '@babel/types': 7.25.6 convert-source-map: 2.0.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -12988,7 +13005,7 @@ snapshots: '@babel/traverse': 7.25.7 '@babel/types': 7.25.8 convert-source-map: 2.0.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -13008,7 +13025,7 @@ snapshots: '@babel/traverse': 7.26.5 '@babel/types': 7.26.5 convert-source-map: 2.0.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -13076,21 +13093,14 @@ snapshots: '@babel/helper-optimise-call-expression': 7.24.7 '@babel/helper-replace-supers': 7.25.0(@babel/core@7.25.2) '@babel/helper-skip-transparent-expression-wrappers': 7.24.7 - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.25.6(supports-color@9.4.0) semver: 6.3.1 transitivePeerDependencies: - supports-color '@babel/helper-member-expression-to-functions@7.24.8': dependencies: - '@babel/traverse': 7.25.6 - '@babel/types': 7.25.6 - transitivePeerDependencies: - - supports-color - - '@babel/helper-module-imports@7.24.7': - dependencies: - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.25.6(supports-color@9.4.0) '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -13130,10 +13140,10 @@ snapshots: '@babel/helper-module-transforms@7.25.2(@babel/core@7.25.2)': dependencies: '@babel/core': 7.25.2 - '@babel/helper-module-imports': 7.24.7 - '@babel/helper-simple-access': 7.24.7 + '@babel/helper-module-imports': 7.24.7(supports-color@9.4.0) + '@babel/helper-simple-access': 7.24.7(supports-color@9.4.0) '@babel/helper-validator-identifier': 7.24.7 - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.25.6(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -13168,14 +13178,7 @@ snapshots: '@babel/core': 7.25.2 '@babel/helper-member-expression-to-functions': 7.24.8 '@babel/helper-optimise-call-expression': 7.24.7 - '@babel/traverse': 7.25.6 - transitivePeerDependencies: - - supports-color - - '@babel/helper-simple-access@7.24.7': - dependencies: - '@babel/traverse': 7.25.6 - '@babel/types': 7.25.6 + '@babel/traverse': 7.25.6(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -13195,7 +13198,7 @@ snapshots: '@babel/helper-skip-transparent-expression-wrappers@7.24.7': dependencies: - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.25.6(supports-color@9.4.0) '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -13430,7 +13433,7 @@ snapshots: '@babel/core': 7.25.2 '@babel/helper-module-transforms': 7.25.2(@babel/core@7.25.2) '@babel/helper-plugin-utils': 7.24.8 - '@babel/helper-simple-access': 7.24.7 + '@babel/helper-simple-access': 7.24.7(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -13484,18 +13487,6 @@ snapshots: '@babel/parser': 7.25.9 '@babel/types': 7.25.9 - '@babel/traverse@7.25.6': - dependencies: - '@babel/code-frame': 7.24.7 - '@babel/generator': 7.25.6 - '@babel/parser': 7.25.6 - '@babel/template': 7.25.0 - '@babel/types': 7.25.6 - debug: 4.4.0(supports-color@8.1.1) - globals: 11.12.0 - transitivePeerDependencies: - - supports-color - '@babel/traverse@7.25.6(supports-color@9.4.0)': dependencies: '@babel/code-frame': 7.24.7 @@ -13515,7 +13506,7 @@ snapshots: '@babel/parser': 7.25.8 '@babel/template': 7.25.9 '@babel/types': 7.25.8 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -13527,7 +13518,7 @@ snapshots: '@babel/parser': 7.26.5 '@babel/template': 7.25.9 '@babel/types': 7.26.5 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -13617,7 +13608,7 @@ snapshots: awaiting: 3.0.0 cheerio: 1.0.0-rc.12 csv-parse: 5.5.6 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -13631,7 +13622,7 @@ snapshots: '@cocalc/primus-responder@1.0.5': dependencies: - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) node-uuid: 1.4.8 transitivePeerDependencies: - supports-color @@ -13904,7 +13895,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -14066,7 +14057,7 @@ snapshots: '@humanwhocodes/config-array@0.13.0': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -14090,7 +14081,7 @@ snapshots: '@antfu/install-pkg': 0.4.1 '@antfu/utils': 0.7.10 '@iconify/types': 2.0.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) kolorist: 1.8.0 local-pkg: 0.5.0 mlly: 1.7.2 @@ -14294,7 +14285,7 @@ snapshots: istanbul-lib-coverage: 3.2.2 istanbul-lib-instrument: 6.0.3 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 4.0.1 + istanbul-lib-source-maps: 4.0.1(supports-color@9.4.0) istanbul-reports: 3.1.7 jest-message-util: 29.7.0 jest-util: 29.7.0 @@ -14840,6 +14831,21 @@ snapshots: '@module-federation/runtime': 0.5.1 '@module-federation/sdk': 0.5.1 + '@nats-io/jetstream@3.0.0-36': + dependencies: + '@nats-io/nats-core': 3.0.0-49 + + '@nats-io/nats-core@3.0.0-49': + dependencies: + '@nats-io/nkeys': 2.0.2 + '@nats-io/nuid': 2.0.3 + + '@nats-io/nkeys@2.0.2': + dependencies: + tweetnacl: 1.0.3 + + '@nats-io/nuid@2.0.3': {} + '@nestjs/axios@3.0.3(@nestjs/common@10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1))(axios@1.7.4)(rxjs@7.8.1)': dependencies: '@nestjs/common': 10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1) @@ -14905,7 +14911,7 @@ snapshots: '@types/xml-encryption': 1.2.4 '@types/xml2js': 0.4.14 '@xmldom/xmldom': 0.8.10 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) xml-crypto: 3.2.0 xml-encryption: 3.0.2 xml2js: 0.5.0 @@ -16006,7 +16012,7 @@ snapshots: '@typescript-eslint/type-utils': 6.21.0(eslint@8.57.1)(typescript@5.7.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.1)(typescript@5.7.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) eslint: 8.57.1 graphemer: 1.4.0 ignore: 5.3.1 @@ -16024,7 +16030,7 @@ snapshots: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.7.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) eslint: 8.57.1 optionalDependencies: typescript: 5.7.3 @@ -16040,7 +16046,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.7.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.1)(typescript@5.7.3) - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) eslint: 8.57.1 ts-api-utils: 1.3.0(typescript@5.7.3) optionalDependencies: @@ -16054,7 +16060,7 @@ snapshots: dependencies: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 @@ -16257,13 +16263,13 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) transitivePeerDependencies: - supports-color agent-base@7.1.1: dependencies: - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -18614,7 +18620,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -19000,11 +19006,11 @@ snapshots: follow-redirects@1.15.6(debug@4.4.0): optionalDependencies: - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) follow-redirects@1.15.9(debug@4.4.0): optionalDependencies: - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) font-atlas@2.1.0: dependencies: @@ -19808,7 +19814,7 @@ snapshots: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -19835,21 +19841,21 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.2: dependencies: agent-base: 7.1.1 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.4: dependencies: agent-base: 7.1.1 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -19872,7 +19878,7 @@ snapshots: '@types/tough-cookie': 4.0.5 axios: 1.7.4(debug@4.4.0) camelcase: 6.3.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) dotenv: 16.4.7 extend: 3.0.2 file-type: 16.5.4 @@ -20303,14 +20309,6 @@ snapshots: make-dir: 4.0.0 supports-color: 7.2.0 - istanbul-lib-source-maps@4.0.1: - dependencies: - debug: 4.4.0(supports-color@8.1.1) - istanbul-lib-coverage: 3.2.2 - source-map: 0.6.1 - transitivePeerDependencies: - - supports-color - istanbul-lib-source-maps@4.0.1(supports-color@9.4.0): dependencies: debug: 4.4.0(supports-color@9.4.0) @@ -21883,7 +21881,7 @@ snapshots: istanbul-lib-instrument: 4.0.3 istanbul-lib-processinfo: 2.0.3 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 4.0.1 + istanbul-lib-source-maps: 4.0.1(supports-color@9.4.0) istanbul-reports: 3.1.7 make-dir: 3.1.0 node-preload: 0.2.1 @@ -24177,7 +24175,7 @@ snapshots: spdy-transport@3.0.0: dependencies: - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) detect-node: 2.1.0 hpack.js: 2.1.6 obuf: 1.1.2 @@ -24188,7 +24186,7 @@ snapshots: spdy@4.0.2: dependencies: - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) handle-thing: 2.0.1 http-deceiver: 1.2.7 select-hose: 2.0.0 @@ -25336,7 +25334,7 @@ snapshots: dependencies: '@wwa/statvfs': 1.1.18 awaiting: 3.0.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0(supports-color@9.4.0) port-get: 1.0.4 ws: 8.18.0 transitivePeerDependencies: diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts index d6e4701dd1..6fb959f338 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/nats/index.ts @@ -88,7 +88,7 @@ async function getResponse({ endpoint, params, nc }) { case "restart-terminal": return await restartTerminal(params); case "write-to-terminal": - return writeToTerminal(params); + return await writeToTerminal(params); default: throw Error(`unknown endpoint '${endpoint}'`); } diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index 6d8400c3d2..bc37bc347f 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -11,6 +11,7 @@ import { project_id } from "@cocalc/project/data"; import { sha1 } from "@cocalc/backend/sha1"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { JSONCodec } from "nats"; +const EXIT_MESSAGE = "\r\n\r\n[Process completed - press any key]\r\n\r\n"; const DEFAULT_COMMAND = "/bin/bash"; const jc = JSONCodec(); @@ -39,12 +40,12 @@ export const createTerminal = reuseInFlight( }, ); -export function writeToTerminal({ data, path }: { data; path }) { +export async function writeToTerminal({ data, path }: { data; path }) { const terminal = sessions[path]; if (terminal == null) { throw Error(`no terminal session '${path}'`); } - terminal.write(data); + await terminal.write(data); return { success: true }; } @@ -65,6 +66,7 @@ class Session { private size?: { rows: number; cols: number }; // the subject where we publish our output public subject: string; + private state: "running" | "off" = "off"; constructor({ path, options, nc }) { this.nc = nc; @@ -73,12 +75,12 @@ class Session { this.subject = `project.${project_id}.terminal.${sha1(path)}`; } - write = (data) => { + write = async (data) => { console.log("write", { data }); - if (this.pty == null) { - return; + if (this.state == "off") { + await this.restart(); } - this.pty.write(data); + this.pty?.write(data); }; restart = async () => { @@ -109,13 +111,18 @@ class Session { rows: this.size?.rows, cols: this.size?.cols, }); - + this.state = "running"; this.pty.onData((data) => { // console.log("onData", { data }); this.nc.publish(this.subject, jc.encode({ data })); }); this.pty.onExit((status) => { + this.nc.publish( + this.subject, + jc.encode({ data: EXIT_MESSAGE }), + ); this.nc.publish(this.subject, jc.encode({ ...status, exit: true })); + this.state = "off"; }); }; } diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index a5d60fea89..4a248407c6 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -130,6 +130,9 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { goalPub.add(`project.${userId}.>`); goalSub.add(`project.${userId}.>`); } + // TEMPORARY: for learsning jetstream! + goalPub.add("$JS.>"); + goalSub.add("$JS.>"); // **Subject Permissions SYNC Algorithm ** // figure out what signing key currently allows an update it to be exactly what is specified above. From 277b139b7dfd0748304b97f3943ce919e8fb8892 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 23 Jan 2025 01:47:22 +0000 Subject: [PATCH 018/281] nats - move terminal jetstream creation to project; avoid junk characters --- docs/nats/devlog.md | 6 +- src/packages/frontend/client/nats.ts | 6 +- .../terminal-editor/connected-terminal.ts | 6 +- .../nats-terminal-connection.ts | 56 +++++--- src/packages/pnpm-lock.yaml | 133 +++++++++++------- src/packages/project/nats/terminal.ts | 38 +++-- src/packages/project/package.json | 1 + 7 files changed, 163 insertions(+), 83 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index ea0002d7a0..5f21207f02 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -219,10 +219,10 @@ NOTE!!! The above consumer is ephemeral -- it disappears if we don't grab it via ## [ ] Goal: Jetstream permissions -- [ ] who sets up the stream for capturing terminal outputs and when? -- [ ] what are the permissions for jetstream usage and access? -- [ ] deleting old data? +- [ ] project should set up the stream for capturing terminal outputs. +- [ ] delete old messages with a given subject. `nats stream purge project-81e0c408-ac65-4114-bad5-5f4b6539bd0e-terminal --seq=7000` - [ ] handle the other messages like resize +- [ ] permissions for jetstream usage and access ## [ ] Goal: Terminal and **compute server** diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 44a43a8406..883790246c 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -86,7 +86,11 @@ export class NatsClient { params, }), ); - return this.jc.decode(resp.data); + const x = this.jc.decode(resp.data) as any; + if (x?.error) { + throw Error(x.error); + } + return x; }; // for debugging -- listen to and display all messages on a subject diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index fe2ea65387..8827bfc6cb 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -281,9 +281,9 @@ export class Terminal { project_id: this.project_id, }); this.conn = conn as any; - conn?.on("close", this.connect); - conn?.on("data", this._handle_data_from_project); - await conn?.init(); + conn.on("close", this.connect); + conn.on("data", this._handle_data_from_project); + await conn.init(); this.set_connection_status("connected"); } catch (err) { this.set_connection_status("disconnected"); diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 1a0b11d651..6ed80ec90e 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -2,12 +2,15 @@ import { webapp_client } from "@cocalc/frontend/webapp-client"; import { EventEmitter } from "events"; import { JSONCodec } from "nats.ws"; import sha1 from "sha1"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; export class NatsTerminalConnection extends EventEmitter { private project_id: string; private path: string; private subject: string; - private state: null | "running" | "off"; + private state: null | "running" | "off" | "closed"; + private consumer?; + private startInit: number = 0; constructor({ project_id, path }) { super(); @@ -18,6 +21,11 @@ export class NatsTerminalConnection extends EventEmitter { } write = async (data) => { + if (Date.now() - this.startInit <= 2000) { + // ignore initial data while initializing (e.g., first 2 seconds for now -- TODO: use nats more cleverly) + // This is the trickt to avoid "junk characters" on refresh/reconnect. + return; + } if (this.state != "running") { await this.start(); } @@ -27,11 +35,19 @@ export class NatsTerminalConnection extends EventEmitter { return; } const write = async () => { - await webapp_client.nats_client.project({ - project_id: this.project_id, - endpoint: "write-to-terminal", - params: { path: this.path, data }, - }); + const f = async () => { + await webapp_client.nats_client.project({ + project_id: this.project_id, + endpoint: "write-to-terminal", + params: { path: this.path, data }, + }); + }; + try { + await f(); + } catch (_err) { + await this.start(); + await f(); + } }; try { await write(); @@ -43,16 +59,17 @@ export class NatsTerminalConnection extends EventEmitter { end = () => { // todo + this.state = "closed"; }; - private start = async () => { + private start = reuseInFlight(async () => { // ensure running: await webapp_client.nats_client.project({ project_id: this.project_id, endpoint: "create-terminal", params: { path: this.path }, }); - }; + }); private getConsumer = async () => { // TODO: idempotent, but move to project @@ -62,11 +79,6 @@ export class NatsTerminalConnection extends EventEmitter { const js = nats_client.jetstream.jetstream(nc); // consumer doesn't exist, so setup everything. const jsm = await nats_client.jetstream.jetstreamManager(nc); - await jsm.streams.add({ - name: stream, - subjects: [`project.${this.project_id}.terminal.>`], - compression: "s2", - }); // making an ephemeral consumer for just one subject (e.g., this terminal frame) const { name } = await jsm.consumers.add(stream, { filter_subject: this.subject, @@ -75,10 +87,22 @@ export class NatsTerminalConnection extends EventEmitter { }; init = async () => { - const jc = JSONCodec(); - const consumer = await this.getConsumer(); await this.start(); - for await (const mesg of await consumer.consume()) { + this.consumer = await this.getConsumer(); + this.run(); + }; + + private run = async () => { + if (this.consumer == null) { + return; + } + const jc = JSONCodec(); + // this loop runs forever (or until state = closed or this.consumer.closed())... + this.startInit = Date.now(); + for await (const mesg of await this.consumer.consume()) { + if (this.state == "closed") { + return; + } const { exit, data } = jc.decode(mesg.data) as any; if (exit) { this.state = "off"; diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index deb3c2f072..4e5d2f41e1 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -191,7 +191,7 @@ importers: version: 8.7.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@9.4.0) + version: 4.4.0(supports-color@8.1.1) immutable: specifier: ^4.3.0 version: 4.3.7 @@ -399,7 +399,7 @@ importers: version: 1.11.13 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@9.4.0) + version: 4.4.0(supports-color@8.1.1) direction: specifier: ^1.0.4 version: 1.0.4 @@ -802,7 +802,7 @@ importers: version: 2.8.5 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@9.4.0) + version: 4.4.0(supports-color@8.1.1) escape-html: specifier: ^1.0.3 version: 1.0.3 @@ -980,7 +980,7 @@ importers: version: 8.7.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@9.4.0) + version: 4.4.0(supports-color@8.1.1) enchannel-zmq-backend: specifier: ^9.1.23 version: 9.1.23(rxjs@7.8.1) @@ -1226,6 +1226,9 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util + '@nats-io/jetstream': + specifier: 3.0.0-36 + version: 3.0.0-36 '@nteract/messaging': specifier: ^7.0.20 version: 7.0.20 @@ -1255,7 +1258,7 @@ importers: version: 3.0.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@9.4.0) + version: 4.4.0(supports-color@8.1.1) diskusage: specifier: ^1.1.3 version: 1.2.0 @@ -1840,7 +1843,7 @@ importers: version: 3.0.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@9.4.0) + version: 4.4.0(supports-color@8.1.1) events: specifier: 3.3.0 version: 3.3.0 @@ -1892,7 +1895,7 @@ importers: version: 1.0.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@9.4.0) + version: 4.4.0(supports-color@8.1.1) primus: specifier: ^8.0.9 version: 8.0.9 @@ -1972,7 +1975,7 @@ importers: version: 3.0.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@9.4.0) + version: 4.4.0(supports-color@8.1.1) lodash: specifier: ^4.17.21 version: 4.17.21 @@ -2018,7 +2021,7 @@ importers: version: 1.11.13 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@9.4.0) + version: 4.4.0(supports-color@8.1.1) decimal.js-light: specifier: ^2.5.1 version: 2.5.1 @@ -12962,10 +12965,10 @@ snapshots: '@babel/helpers': 7.25.6 '@babel/parser': 7.25.6 '@babel/template': 7.25.0 - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 '@babel/types': 7.25.6 convert-source-map: 2.0.0 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -13005,7 +13008,7 @@ snapshots: '@babel/traverse': 7.25.7 '@babel/types': 7.25.8 convert-source-map: 2.0.0 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -13025,7 +13028,7 @@ snapshots: '@babel/traverse': 7.26.5 '@babel/types': 7.26.5 convert-source-map: 2.0.0 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -13093,14 +13096,21 @@ snapshots: '@babel/helper-optimise-call-expression': 7.24.7 '@babel/helper-replace-supers': 7.25.0(@babel/core@7.25.2) '@babel/helper-skip-transparent-expression-wrappers': 7.24.7 - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 semver: 6.3.1 transitivePeerDependencies: - supports-color '@babel/helper-member-expression-to-functions@7.24.8': dependencies: - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 + '@babel/types': 7.25.6 + transitivePeerDependencies: + - supports-color + + '@babel/helper-module-imports@7.24.7': + dependencies: + '@babel/traverse': 7.25.6 '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -13140,10 +13150,10 @@ snapshots: '@babel/helper-module-transforms@7.25.2(@babel/core@7.25.2)': dependencies: '@babel/core': 7.25.2 - '@babel/helper-module-imports': 7.24.7(supports-color@9.4.0) - '@babel/helper-simple-access': 7.24.7(supports-color@9.4.0) + '@babel/helper-module-imports': 7.24.7 + '@babel/helper-simple-access': 7.24.7 '@babel/helper-validator-identifier': 7.24.7 - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 transitivePeerDependencies: - supports-color @@ -13178,7 +13188,14 @@ snapshots: '@babel/core': 7.25.2 '@babel/helper-member-expression-to-functions': 7.24.8 '@babel/helper-optimise-call-expression': 7.24.7 - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 + transitivePeerDependencies: + - supports-color + + '@babel/helper-simple-access@7.24.7': + dependencies: + '@babel/traverse': 7.25.6 + '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -13198,7 +13215,7 @@ snapshots: '@babel/helper-skip-transparent-expression-wrappers@7.24.7': dependencies: - '@babel/traverse': 7.25.6(supports-color@9.4.0) + '@babel/traverse': 7.25.6 '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -13433,7 +13450,7 @@ snapshots: '@babel/core': 7.25.2 '@babel/helper-module-transforms': 7.25.2(@babel/core@7.25.2) '@babel/helper-plugin-utils': 7.24.8 - '@babel/helper-simple-access': 7.24.7(supports-color@9.4.0) + '@babel/helper-simple-access': 7.24.7 transitivePeerDependencies: - supports-color @@ -13487,6 +13504,18 @@ snapshots: '@babel/parser': 7.25.9 '@babel/types': 7.25.9 + '@babel/traverse@7.25.6': + dependencies: + '@babel/code-frame': 7.24.7 + '@babel/generator': 7.25.6 + '@babel/parser': 7.25.6 + '@babel/template': 7.25.0 + '@babel/types': 7.25.6 + debug: 4.4.0(supports-color@8.1.1) + globals: 11.12.0 + transitivePeerDependencies: + - supports-color + '@babel/traverse@7.25.6(supports-color@9.4.0)': dependencies: '@babel/code-frame': 7.24.7 @@ -13506,7 +13535,7 @@ snapshots: '@babel/parser': 7.25.8 '@babel/template': 7.25.9 '@babel/types': 7.25.8 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -13518,7 +13547,7 @@ snapshots: '@babel/parser': 7.26.5 '@babel/template': 7.25.9 '@babel/types': 7.26.5 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -13608,7 +13637,7 @@ snapshots: awaiting: 3.0.0 cheerio: 1.0.0-rc.12 csv-parse: 5.5.6 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -13622,7 +13651,7 @@ snapshots: '@cocalc/primus-responder@1.0.5': dependencies: - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) node-uuid: 1.4.8 transitivePeerDependencies: - supports-color @@ -13895,7 +13924,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -14057,7 +14086,7 @@ snapshots: '@humanwhocodes/config-array@0.13.0': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -14081,7 +14110,7 @@ snapshots: '@antfu/install-pkg': 0.4.1 '@antfu/utils': 0.7.10 '@iconify/types': 2.0.0 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) kolorist: 1.8.0 local-pkg: 0.5.0 mlly: 1.7.2 @@ -14285,7 +14314,7 @@ snapshots: istanbul-lib-coverage: 3.2.2 istanbul-lib-instrument: 6.0.3 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 4.0.1(supports-color@9.4.0) + istanbul-lib-source-maps: 4.0.1 istanbul-reports: 3.1.7 jest-message-util: 29.7.0 jest-util: 29.7.0 @@ -14911,7 +14940,7 @@ snapshots: '@types/xml-encryption': 1.2.4 '@types/xml2js': 0.4.14 '@xmldom/xmldom': 0.8.10 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) xml-crypto: 3.2.0 xml-encryption: 3.0.2 xml2js: 0.5.0 @@ -16012,7 +16041,7 @@ snapshots: '@typescript-eslint/type-utils': 6.21.0(eslint@8.57.1)(typescript@5.7.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.1)(typescript@5.7.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) eslint: 8.57.1 graphemer: 1.4.0 ignore: 5.3.1 @@ -16030,7 +16059,7 @@ snapshots: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.7.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) eslint: 8.57.1 optionalDependencies: typescript: 5.7.3 @@ -16046,7 +16075,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.7.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.1)(typescript@5.7.3) - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) eslint: 8.57.1 ts-api-utils: 1.3.0(typescript@5.7.3) optionalDependencies: @@ -16060,7 +16089,7 @@ snapshots: dependencies: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 @@ -16263,13 +16292,13 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color agent-base@7.1.1: dependencies: - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -18620,7 +18649,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -19006,11 +19035,11 @@ snapshots: follow-redirects@1.15.6(debug@4.4.0): optionalDependencies: - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) follow-redirects@1.15.9(debug@4.4.0): optionalDependencies: - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) font-atlas@2.1.0: dependencies: @@ -19814,7 +19843,7 @@ snapshots: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -19841,21 +19870,21 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.2: dependencies: agent-base: 7.1.1 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.4: dependencies: agent-base: 7.1.1 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -19878,7 +19907,7 @@ snapshots: '@types/tough-cookie': 4.0.5 axios: 1.7.4(debug@4.4.0) camelcase: 6.3.0 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) dotenv: 16.4.7 extend: 3.0.2 file-type: 16.5.4 @@ -20309,6 +20338,14 @@ snapshots: make-dir: 4.0.0 supports-color: 7.2.0 + istanbul-lib-source-maps@4.0.1: + dependencies: + debug: 4.4.0(supports-color@8.1.1) + istanbul-lib-coverage: 3.2.2 + source-map: 0.6.1 + transitivePeerDependencies: + - supports-color + istanbul-lib-source-maps@4.0.1(supports-color@9.4.0): dependencies: debug: 4.4.0(supports-color@9.4.0) @@ -21881,7 +21918,7 @@ snapshots: istanbul-lib-instrument: 4.0.3 istanbul-lib-processinfo: 2.0.3 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 4.0.1(supports-color@9.4.0) + istanbul-lib-source-maps: 4.0.1 istanbul-reports: 3.1.7 make-dir: 3.1.0 node-preload: 0.2.1 @@ -24175,7 +24212,7 @@ snapshots: spdy-transport@3.0.0: dependencies: - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) detect-node: 2.1.0 hpack.js: 2.1.6 obuf: 1.1.2 @@ -24186,7 +24223,7 @@ snapshots: spdy@4.0.2: dependencies: - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) handle-thing: 2.0.1 http-deceiver: 1.2.7 select-hose: 2.0.0 @@ -25334,7 +25371,7 @@ snapshots: dependencies: '@wwa/statvfs': 1.1.18 awaiting: 3.0.0 - debug: 4.4.0(supports-color@9.4.0) + debug: 4.4.0(supports-color@8.1.1) port-get: 1.0.4 ws: 8.18.0 transitivePeerDependencies: diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index bc37bc347f..9e8c552e74 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -11,8 +11,9 @@ import { project_id } from "@cocalc/project/data"; import { sha1 } from "@cocalc/backend/sha1"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { JSONCodec } from "nats"; -const EXIT_MESSAGE = "\r\n\r\n[Process completed - press any key]\r\n\r\n"; +import { /*jetstream,*/ jetstreamManager } from "@nats-io/jetstream"; +const EXIT_MESSAGE = "\r\n\r\n[Process completed - press any key]\r\n\r\n"; const DEFAULT_COMMAND = "/bin/bash"; const jc = JSONCodec(); @@ -61,18 +62,20 @@ export async function restartTerminal({ path }: { path }) { class Session { private nc; private path: string; - private options?; + private options; private pty?; private size?: { rows: number; cols: number }; // the subject where we publish our output public subject: string; private state: "running" | "off" = "off"; + private streamName: string; constructor({ path, options, nc }) { this.nc = nc; this.path = path; - this.options = options; + this.options = options ?? {}; this.subject = `project.${project_id}.terminal.${sha1(path)}`; + this.streamName = `project-${project_id}-terminal`; } write = async (data) => { @@ -89,22 +92,33 @@ class Session { await this.init(); }; + getStream = async () => { + // idempotent so don't have to check if there is already a stream + const nc = this.nc; + const jsm = await jetstreamManager(nc); + await jsm.streams.add({ + name: this.streamName, + subjects: [`project.${project_id}.terminal.>`], + compression: "s2", + }); + }; + init = async () => { const { head, tail } = path_split(this.path); const env = { COCALC_TERMINAL_FILENAME: tail, ...envForSpawn(), - ...this.options?.env, + ...this.options.env, TMUX: undefined, // ensure not set }; - const command = this.options?.command ?? DEFAULT_COMMAND; - const args = this.options?.args ?? []; + const command = this.options.command ?? DEFAULT_COMMAND; + const args = this.options.args ?? []; const initFilename: string = console_init_filename(this.path); if (await exists(initFilename)) { args.push("--init-file"); args.push(path_split(initFilename).tail); } - const cwd = getCWD(head, this.options?.cwd); + const cwd = getCWD(head, this.options.cwd); this.pty = spawn(command, args, { cwd, env, @@ -112,15 +126,15 @@ class Session { cols: this.size?.cols, }); this.state = "running"; - this.pty.onData((data) => { + await this.getStream(); + //const js = await jetstream(this.nc); + this.pty.onData(async (data) => { // console.log("onData", { data }); + //await js.publish(this.streamName, jc.encode({ data })); this.nc.publish(this.subject, jc.encode({ data })); }); this.pty.onExit((status) => { - this.nc.publish( - this.subject, - jc.encode({ data: EXIT_MESSAGE }), - ); + this.nc.publish(this.subject, jc.encode({ data: EXIT_MESSAGE })); this.nc.publish(this.subject, jc.encode({ ...status, exit: true })); this.state = "off"; }); diff --git a/src/packages/project/package.json b/src/packages/project/package.json index 46d13ecb22..81dd538103 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -31,6 +31,7 @@ "@cocalc/sync-fs": "workspace:*", "@cocalc/terminal": "workspace:*", "@cocalc/util": "workspace:*", + "@nats-io/jetstream": "3.0.0-36", "@nteract/messaging": "^7.0.20", "@types/lodash": "^4.14.202", "@types/primus": "^7.3.9", From 362568ef22052859df8f137ed329573aed8d6a46 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 23 Jan 2025 02:13:14 +0000 Subject: [PATCH 019/281] remove manifest in DEBUG mode (since it's constantly failing and the log is annoying) --- src/packages/static/src/manifest.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/packages/static/src/manifest.tsx b/src/packages/static/src/manifest.tsx index add8faadfa..fd4d306132 100644 --- a/src/packages/static/src/manifest.tsx +++ b/src/packages/static/src/manifest.tsx @@ -19,7 +19,12 @@ window.addEventListener("load", async function () { } }); +declare var DEBUG; + export default function Manifest() { + if (DEBUG) { + return null; + } return ( Date: Thu, 23 Jan 2025 03:17:31 +0000 Subject: [PATCH 020/281] NATS: reimplement terminal initialization... yet again! --- docs/nats/devlog.md | 5 +- .../nats-terminal-connection.ts | 89 +++++++++++-------- src/packages/project/nats/terminal.ts | 46 +++++++--- 3 files changed, 90 insertions(+), 50 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 5f21207f02..958e06546f 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -219,8 +219,9 @@ NOTE!!! The above consumer is ephemeral -- it disappears if we don't grab it via ## [ ] Goal: Jetstream permissions -- [ ] project should set up the stream for capturing terminal outputs. -- [ ] delete old messages with a given subject. `nats stream purge project-81e0c408-ac65-4114-bad5-5f4b6539bd0e-terminal --seq=7000` +- [x] project should set up the stream for capturing terminal outputs. +- [x] delete old messages with a given subject. `nats stream purge project-81e0c408-ac65-4114-bad5-5f4b6539bd0e-terminal --seq=7000` + - there is a setting max\_msgs\_per\_subject on a stream, so **we just set that and are done!** Gees. It is too easy. - [ ] handle the other messages like resize - [ ] permissions for jetstream usage and access diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 6ed80ec90e..3b89a0a6bc 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -4,25 +4,38 @@ import { JSONCodec } from "nats.ws"; import sha1 from "sha1"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +const jc = JSONCodec(); + export class NatsTerminalConnection extends EventEmitter { private project_id: string; private path: string; private subject: string; - private state: null | "running" | "off" | "closed"; + private state: null | "running" | "init" | "closed"; private consumer?; - private startInit: number = 0; + // keep = optional number of messages to retain between clients/sessions/view, i.e., + // "amount of history". This is global to all terminals in the project. + private keep?: number; - constructor({ project_id, path }) { + constructor({ + project_id, + path, + keep, + }: { + project_id: string; + path: string; + keep?: number; + }) { super(); this.project_id = project_id; this.path = path; + this.keep = keep; // move to util so guaranteed in sync with project this.subject = `project.${project_id}.terminal.${sha1(path)}`; } write = async (data) => { - if (Date.now() - this.startInit <= 2000) { - // ignore initial data while initializing (e.g., first 2 seconds for now -- TODO: use nats more cleverly) + if (this.state == "init") { + // ignore initial data while initializing. // This is the trickt to avoid "junk characters" on refresh/reconnect. return; } @@ -30,30 +43,22 @@ export class NatsTerminalConnection extends EventEmitter { await this.start(); } if (typeof data != "string") { - //console.log("write -- todo:", data); - // TODO: not yet implemented, e.g., {cmd: 'size', rows: 18, cols: 180} + console.log(data); return; } - const write = async () => { - const f = async () => { - await webapp_client.nats_client.project({ - project_id: this.project_id, - endpoint: "write-to-terminal", - params: { path: this.path, data }, - }); - }; - try { - await f(); - } catch (_err) { - await this.start(); - await f(); - } + const f = async () => { + await webapp_client.nats_client.project({ + project_id: this.project_id, + endpoint: "write-to-terminal", + params: { path: this.path, data, keep: this.keep }, + }); }; + try { - await write(); - } catch (_) { + await f(); + } catch (_err) { await this.start(); - await write(); + await f(); } }; @@ -87,28 +92,42 @@ export class NatsTerminalConnection extends EventEmitter { }; init = async () => { + this.state = "init"; await this.start(); this.consumer = await this.getConsumer(); this.run(); }; + private handle = (mesg) => { + if (this.state == "closed") { + return true; + } + const { data } = jc.decode(mesg.data) as any; + if (data != null) { + this.emit("data", data); + } + }; + private run = async () => { if (this.consumer == null) { return; } - const jc = JSONCodec(); - // this loop runs forever (or until state = closed or this.consumer.closed())... - this.startInit = Date.now(); - for await (const mesg of await this.consumer.consume()) { - if (this.state == "closed") { + const messages = await this.consumer.fetch({ + max_messages: this.keep, + expires: 1000, + }); + for await (const mesg of messages) { + if (this.handle(mesg)) { return; } - const { exit, data } = jc.decode(mesg.data) as any; - if (exit) { - this.state = "off"; - } else if (data != null) { - this.state = "running"; - this.emit("data", data); + } + if (this.state == "init") { + this.state = "running"; + } + // TODO: this loop runs until state = closed or this.consumer.closed()... ? + for await (const mesg of await this.consumer.consume()) { + if (this.handle(mesg)) { + return; } } }; diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index 9e8c552e74..2fdb18ef24 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -1,5 +1,7 @@ /* -Quick very simple terminal proof of concept for testing NATS +Terminal + +- using NATS */ import { spawn } from "node-pty"; @@ -11,8 +13,14 @@ import { project_id } from "@cocalc/project/data"; import { sha1 } from "@cocalc/backend/sha1"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { JSONCodec } from "nats"; -import { /*jetstream,*/ jetstreamManager } from "@nats-io/jetstream"; +import { jetstreamManager } from "@nats-io/jetstream"; +import { getLogger } from "@cocalc/project/logger"; + +const logger = getLogger("server:nats:terminal"); +const DEFAULT_KEEP = 300; +const MIN_KEEP = 5; +const MAX_KEEP = 2000; const EXIT_MESSAGE = "\r\n\r\n[Process completed - press any key]\r\n\r\n"; const DEFAULT_COMMAND = "/bin/bash"; const jc = JSONCodec(); @@ -24,7 +32,7 @@ export const createTerminal = reuseInFlight( if (params == null) { throw Error("params must be specified"); } - const { path, options } = params; + const { path, ...options } = params; if (!path) { throw Error("path must be specified"); } @@ -69,17 +77,22 @@ class Session { public subject: string; private state: "running" | "off" = "off"; private streamName: string; + private keep: number; constructor({ path, options, nc }) { + logger.debug("create session ", { path, options }); this.nc = nc; this.path = path; - this.options = options ?? {}; + this.options = options; + this.keep = Math.max( + MIN_KEEP, + Math.min(this.options.keep ?? DEFAULT_KEEP, MAX_KEEP), + ); this.subject = `project.${project_id}.terminal.${sha1(path)}`; this.streamName = `project-${project_id}-terminal`; } write = async (data) => { - console.log("write", { data }); if (this.state == "off") { await this.restart(); } @@ -96,11 +109,21 @@ class Session { // idempotent so don't have to check if there is already a stream const nc = this.nc; const jsm = await jetstreamManager(nc); - await jsm.streams.add({ - name: this.streamName, - subjects: [`project.${project_id}.terminal.>`], - compression: "s2", - }); + try { + await jsm.streams.add({ + name: this.streamName, + subjects: [`project.${project_id}.terminal.>`], + compression: "s2", + max_msgs_per_subject: this.keep, + }); + } catch (_err) { + // probably already exists + await jsm.streams.update(this.streamName, { + subjects: [`project.${project_id}.terminal.>`], + compression: "s2" as any, + max_msgs_per_subject: this.keep, + }); + } }; init = async () => { @@ -127,10 +150,7 @@ class Session { }); this.state = "running"; await this.getStream(); - //const js = await jetstream(this.nc); this.pty.onData(async (data) => { - // console.log("onData", { data }); - //await js.publish(this.streamName, jc.encode({ data })); this.nc.publish(this.subject, jc.encode({ data })); }); this.pty.onExit((status) => { From bdb9053854b7cb3c8becd52aca1dea8a6b8b5594 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 23 Jan 2025 04:03:43 +0000 Subject: [PATCH 021/281] nats terminal -- partial messaging for resizing the terminal --- .../terminal-editor/connected-terminal.ts | 8 ++- .../nats-terminal-connection.ts | 33 ++++++++++-- src/packages/project/nats/index.ts | 9 +++- src/packages/project/nats/terminal.ts | 51 +++++++++++++++++++ 4 files changed, 93 insertions(+), 8 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 8827bfc6cb..d80c478ce7 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -274,7 +274,7 @@ export class Terminal { connectNats = async () => { try { - this.ignore_terminal_data = false; // todo + this.ignore_terminal_data = true; this.set_connection_status("connecting"); const conn = new NatsTerminalConnection({ path: this.term_path, @@ -283,8 +283,11 @@ export class Terminal { this.conn = conn as any; conn.on("close", this.connect); conn.on("data", this._handle_data_from_project); + conn.once("ready", () => { + this.ignore_terminal_data = false; + this.set_connection_status("connected"); + }); await conn.init(); - this.set_connection_status("connected"); } catch (err) { this.set_connection_status("disconnected"); throw err; @@ -856,6 +859,7 @@ export class Terminal { } this.last_geom = { rows, cols }; this.conn_write({ cmd: "size", rows, cols }); + this.terminal_resize({ rows, cols }); } copy(): void { diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 3b89a0a6bc..75eac8b6d7 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -43,7 +43,27 @@ export class NatsTerminalConnection extends EventEmitter { await this.start(); } if (typeof data != "string") { - console.log(data); + console.log("to project", data); + if (data.cmd == "size") { + const { rows, cols } = data; + if ( + rows <= 0 || + cols <= 0 || + rows == Infinity || + cols == Infinity || + isNaN(rows) || + isNaN(cols) + ) { + // invalid measurement -- ignore; https://github.com/sagemathinc/cocalc/issues/4158 and https://github.com/sagemathinc/cocalc/issues/4266 + return; + } + } + const resp = await webapp_client.nats_client.project({ + project_id: this.project_id, + endpoint: "terminal-command", + params: { path: this.path, ...data }, + }); + console.log("got back ", resp); return; } const f = async () => { @@ -102,9 +122,11 @@ export class NatsTerminalConnection extends EventEmitter { if (this.state == "closed") { return true; } - const { data } = jc.decode(mesg.data) as any; - if (data != null) { - this.emit("data", data); + const x = jc.decode(mesg.data) as any; + if (x?.data != null) { + this.emit("data", x?.data); + } else { + console.log("TODO -- from project:", x); } }; @@ -113,7 +135,7 @@ export class NatsTerminalConnection extends EventEmitter { return; } const messages = await this.consumer.fetch({ - max_messages: this.keep, + max_messages: 10000, expires: 1000, }); for await (const mesg of messages) { @@ -123,6 +145,7 @@ export class NatsTerminalConnection extends EventEmitter { } if (this.state == "init") { this.state = "running"; + this.emit("ready"); } // TODO: this loop runs until state = closed or this.consumer.closed()... ? for await (const mesg of await this.consumer.consume()) { diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts index 6fb959f338..8f2c1dfdb7 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/nats/index.ts @@ -74,7 +74,12 @@ async function handleApiRequest({ mesg, group, account_id, nc }) { mesg.respond(jc.encode(resp)); } -import { createTerminal, restartTerminal, writeToTerminal } from "./terminal"; +import { + createTerminal, + restartTerminal, + terminalCommand, + writeToTerminal, +} from "./terminal"; async function getResponse({ endpoint, params, nc }) { switch (endpoint) { case "ping": @@ -87,6 +92,8 @@ async function getResponse({ endpoint, params, nc }) { return await createTerminal({ params, nc }); case "restart-terminal": return await restartTerminal(params); + case "terminal-command": + return await terminalCommand(params); case "write-to-terminal": return await writeToTerminal(params); default: diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index 2fdb18ef24..0cc2fb4095 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -15,6 +15,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { JSONCodec } from "nats"; import { jetstreamManager } from "@nats-io/jetstream"; import { getLogger } from "@cocalc/project/logger"; +import { readlink, realpath } from "node:fs/promises"; const logger = getLogger("server:nats:terminal"); @@ -67,6 +68,22 @@ export async function restartTerminal({ path }: { path }) { return { success: true }; } +export async function terminalCommand({ path, cmd, ...args }) { + logger.debug("terminalCommand", { path, cmd, args }); + const terminal = sessions[path]; + if (terminal == null) { + throw Error(`no terminal session '${path}'`); + } + switch (cmd) { + case "size": + return terminal.setSize(args as any); + case "cwd": + return await terminal.getCwd(); + default: + throw Error(`unknown cmd="${cmd}"`); + } +} + class Session { private nc; private path: string; @@ -105,6 +122,39 @@ class Session { await this.init(); }; + private getHome = () => { + return process.env.HOME ?? "/home/user"; + }; + + getCwd = async () => { + if (this.pty == null) { + return; + } + // we reply with the current working directory of the underlying terminal process, + // which is why we use readlink and proc below. + const pid = this.pty.pid; + // [hsy/dev] wrapping in realpath, because I had the odd case, where the project's + // home included a symlink, hence the "startsWith" below didn't remove the home dir. + const home = await realpath(this.getHome()); + const cwd = await readlink(`/proc/${pid}/cwd`); + // try to send back a relative path, because the webapp does not + // understand absolute paths + const path = cwd.startsWith(home) ? cwd.slice(home.length + 1) : cwd; + return path; + }; + + setSize = ({ rows, cols }: { rows: number; cols: number }) => { + logger.debug("setSize", { rows, cols }); + if (this.pty == null) { + logger.debug("setSize: not doing since pty not defined"); + return; + } + logger.debug("setSize", { rows, cols }, "DOING IT!"); + + this.pty.resize(cols, rows); + this.size = { rows, cols }; + }; + getStream = async () => { // idempotent so don't have to check if there is already a stream const nc = this.nc; @@ -142,6 +192,7 @@ class Session { args.push(path_split(initFilename).tail); } const cwd = getCWD(head, this.options.cwd); + logger.debug("creating pty with size", this.size); this.pty = spawn(command, args, { cwd, env, From a66fdf6217b278c39bb1cbb2092ee7863f31a4f8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 23 Jan 2025 04:23:24 +0000 Subject: [PATCH 022/281] nats terminal: properly doing resizing --- .../terminal-editor/connected-terminal.ts | 6 +- .../nats-terminal-connection.ts | 17 +++- src/packages/project/nats/terminal.ts | 91 ++++++++++++++++--- 3 files changed, 96 insertions(+), 18 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index d80c478ce7..1b4a2500d6 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -279,6 +279,7 @@ export class Terminal { const conn = new NatsTerminalConnection({ path: this.term_path, project_id: this.project_id, + terminalResize: this.terminal_resize, }); this.conn = conn as any; conn.on("close", this.connect); @@ -598,7 +599,7 @@ export class Terminal { // Try to resize terminal to given number of rows and columns. // This should not throw an exception no matter how wrong the input // actually is. - private terminal_resize(opts: { cols: number; rows: number }): void { + private terminal_resize = (opts: { cols: number; rows: number }) => { // console.log("terminal_resize", opts); // terminal.resize only takes integers, hence the floor; // we use floor to avoid cutting off a line halfway. @@ -628,7 +629,7 @@ export class Terminal { } catch (err) { console.warn("Error resizing terminal", err, rows, cols); } - } + }; // Stop ignoring terminal data... but ONLY once // the render buffer is also empty. @@ -859,7 +860,6 @@ export class Terminal { } this.last_geom = { rows, cols }; this.conn_write({ cmd: "size", rows, cols }); - this.terminal_resize({ rows, cols }); } copy(): void { diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 75eac8b6d7..acb6c89723 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -3,8 +3,10 @@ import { EventEmitter } from "events"; import { JSONCodec } from "nats.ws"; import sha1 from "sha1"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { uuid } from "@cocalc/util/misc"; const jc = JSONCodec(); +const client = uuid(); export class NatsTerminalConnection extends EventEmitter { private project_id: string; @@ -15,19 +17,23 @@ export class NatsTerminalConnection extends EventEmitter { // keep = optional number of messages to retain between clients/sessions/view, i.e., // "amount of history". This is global to all terminals in the project. private keep?: number; + private terminalResize; constructor({ project_id, path, keep, + terminalResize, }: { project_id: string; path: string; keep?: number; + terminalResize; }) { super(); this.project_id = project_id; this.path = path; + this.terminalResize = terminalResize; this.keep = keep; // move to util so guaranteed in sync with project this.subject = `project.${project_id}.terminal.${sha1(path)}`; @@ -61,7 +67,7 @@ export class NatsTerminalConnection extends EventEmitter { const resp = await webapp_client.nats_client.project({ project_id: this.project_id, endpoint: "terminal-command", - params: { path: this.path, ...data }, + params: { path: this.path, ...data, client }, }); console.log("got back ", resp); return; @@ -126,7 +132,14 @@ export class NatsTerminalConnection extends EventEmitter { if (x?.data != null) { this.emit("data", x?.data); } else { - console.log("TODO -- from project:", x); + switch (x.cmd) { + case "size": + this.terminalResize(x); + return; + default: + console.log("TODO -- unhandled message from project:", x); + return; + } } }; diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index 0cc2fb4095..2fb2435ff9 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -7,7 +7,7 @@ Terminal import { spawn } from "node-pty"; import { envForSpawn } from "@cocalc/backend/misc"; import { path_split } from "@cocalc/util/misc"; -import { console_init_filename } from "@cocalc/util/misc"; +import { console_init_filename, len } from "@cocalc/util/misc"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { project_id } from "@cocalc/project/data"; import { sha1 } from "@cocalc/backend/sha1"; @@ -24,6 +24,8 @@ const MIN_KEEP = 5; const MAX_KEEP = 2000; const EXIT_MESSAGE = "\r\n\r\n[Process completed - press any key]\r\n\r\n"; const DEFAULT_COMMAND = "/bin/bash"; +const INFINITY = 999999; + const jc = JSONCodec(); const sessions: { [name: string]: Session } = {}; @@ -143,18 +145,6 @@ class Session { return path; }; - setSize = ({ rows, cols }: { rows: number; cols: number }) => { - logger.debug("setSize", { rows, cols }); - if (this.pty == null) { - logger.debug("setSize: not doing since pty not defined"); - return; - } - logger.debug("setSize", { rows, cols }, "DOING IT!"); - - this.pty.resize(cols, rows); - this.size = { rows, cols }; - }; - getStream = async () => { // idempotent so don't have to check if there is already a stream const nc = this.nc; @@ -210,6 +200,81 @@ class Session { this.state = "off"; }); }; + + private clientSizes = {}; + + setSize = ({ + client, + rows, + cols, + }: { + client: string; + rows: number; + cols: number; + }) => { + this.clientSizes[client] = { rows, cols }; + this.resize(); + }; + + private resize = () => { + if (this.pty == null) { + // nothing to do + return; + } + const size = this.getSize(); + if (size == null) { + return; + } + const { rows, cols } = size; + logger.debug("resize", "new size", rows, cols); + try { + this.setSizePty({ rows, cols }); + // broadcast out new size + this.nc.publish(this.subject, jc.encode({ cmd: "size", rows, cols })); + } catch (err) { + logger.debug("terminal channel -- WARNING: unable to resize term", err); + } + }; + + setSizePty = ({ rows, cols }: { rows: number; cols: number }) => { + logger.debug("setSize", { rows, cols }); + if (this.pty == null) { + logger.debug("setSize: not doing since pty not defined"); + return; + } + logger.debug("setSize", { rows, cols }, "DOING IT!"); + + this.pty.resize(cols, rows); + this.size = { rows, cols }; + }; + + getSize = (): { rows: number; cols: number } | undefined => { + const sizes = this.clientSizes; + if (len(sizes) == 0) { + return; + } + let rows: number = INFINITY; + let cols: number = INFINITY; + for (const id in sizes) { + if (sizes[id].rows) { + // if, since 0 rows or 0 columns means *ignore*. + rows = Math.min(rows, sizes[id].rows); + } + if (sizes[id].cols) { + cols = Math.min(cols, sizes[id].cols); + } + } + if (rows === INFINITY || cols === INFINITY) { + // no clients with known sizes currently visible + return; + } + // ensure valid values + rows = Math.max(rows ?? 1, rows); + cols = Math.max(cols ?? 1, cols); + // cache for future use. + this.size = { rows, cols }; + return { rows, cols }; + }; } function getCWD(pathHead, cwd?): string { From 5a6f48e553745724e8d02fb1253545d63e8ea7bd Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 23 Jan 2025 04:59:00 +0000 Subject: [PATCH 023/281] nats terminal: implement open/close command (not happy about it) --- docs/nats/devlog.md | 1 + .../terminal-editor/connected-terminal.ts | 7 +- .../nats-terminal-connection.ts | 18 +++++ src/packages/project/nats/terminal.ts | 69 +++++++++++++++++-- 4 files changed, 88 insertions(+), 7 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 958e06546f..19cba42f3f 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -223,6 +223,7 @@ NOTE!!! The above consumer is ephemeral -- it disappears if we don't grab it via - [x] delete old messages with a given subject. `nats stream purge project-81e0c408-ac65-4114-bad5-5f4b6539bd0e-terminal --seq=7000` - there is a setting max\_msgs\_per\_subject on a stream, so **we just set that and are done!** Gees. It is too easy. - [ ] handle the other messages like resize +- [ ] need to move those other messages to a different subject that isn't part of the stream!! - [ ] permissions for jetstream usage and access ## [ ] Goal: Terminal and **compute server** diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 1b4a2500d6..f9978d864e 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -280,6 +280,8 @@ export class Terminal { path: this.term_path, project_id: this.project_id, terminalResize: this.terminal_resize, + openPaths: this.open_paths, + closePaths: this.close_paths, }); this.conn = conn as any; conn.on("close", this.connect); @@ -287,6 +289,7 @@ export class Terminal { conn.once("ready", () => { this.ignore_terminal_data = false; this.set_connection_status("connected"); + this.scroll_to_bottom(); }); await conn.init(); } catch (err) { @@ -738,7 +741,7 @@ export class Terminal { project_actions.close_tab(path); } - close_paths(paths: Path[]): void { + close_paths = (paths: Path[]): void => { if (!this.is_visible) { return; } @@ -747,7 +750,7 @@ export class Terminal { this._close_path(x.file); } } - } + }; resize(rows: number, cols: number): void { if (this.terminal.cols === cols && this.terminal.rows === rows) { diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index acb6c89723..d67e21c071 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -18,23 +18,31 @@ export class NatsTerminalConnection extends EventEmitter { // "amount of history". This is global to all terminals in the project. private keep?: number; private terminalResize; + private openPaths; + private closePaths; constructor({ project_id, path, keep, terminalResize, + openPaths, + closePaths, }: { project_id: string; path: string; keep?: number; terminalResize; + openPaths; + closePaths; }) { super(); this.project_id = project_id; this.path = path; this.terminalResize = terminalResize; this.keep = keep; + this.openPaths = openPaths; + this.closePaths = closePaths; // move to util so guaranteed in sync with project this.subject = `project.${project_id}.terminal.${sha1(path)}`; } @@ -136,6 +144,16 @@ export class NatsTerminalConnection extends EventEmitter { case "size": this.terminalResize(x); return; + case "message": + if (this.state != "running") { + return; + } + if (x.payload?.event == "open") { + this.openPaths(x.payload.paths); + } else if (x.payload?.event == "close") { + this.closePaths(x.payload.paths); + } + return; default: console.log("TODO -- unhandled message from project:", x); return; diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index 2fb2435ff9..ce5488cc02 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -191,18 +191,22 @@ class Session { }); this.state = "running"; await this.getStream(); - this.pty.onData(async (data) => { - this.nc.publish(this.subject, jc.encode({ data })); + this.pty.onData((data) => { + this.handleBackendMessages(data); + this.publish({ data }); }); this.pty.onExit((status) => { - this.nc.publish(this.subject, jc.encode({ data: EXIT_MESSAGE })); - this.nc.publish(this.subject, jc.encode({ ...status, exit: true })); + this.publish({ data: EXIT_MESSAGE }); + this.publish({ ...status, exit: true }); this.state = "off"; }); }; - private clientSizes = {}; + private publish = (mesg) => { + this.nc.publish(this.subject, jc.encode(mesg)); + }; + private clientSizes = {}; setSize = ({ client, rows, @@ -275,6 +279,61 @@ class Session { this.size = { rows, cols }; return { rows, cols }; }; + + private backendMessagesBuffer = ""; + private backendMessagesState = "none"; + + private resetBackendMessagesBuffer = () => { + this.backendMessagesBuffer = ""; + this.backendMessagesState = "none"; + }; + + private handleBackendMessages = (data: string) => { + /* parse out messages like this: + \x1b]49;"valid JSON string here"\x07 + and format and send them via our json channel. + NOTE: such messages also get sent via the + normal channel, but ignored by the client. + */ + if (this.backendMessagesState === "none") { + const i = data.indexOf("\x1b]49;"); + if (i == -1) { + return; // nothing to worry about + } + // stringify it so it is easy to see what is there: + this.backendMessagesState = "reading"; + this.backendMessagesBuffer = data.slice(i); + } else { + this.backendMessagesBuffer += data; + } + if (this.backendMessagesBuffer.length >= 6) { + const i = this.backendMessagesBuffer.indexOf("\x07"); + if (i == -1) { + // continue to wait... unless too long + if (this.backendMessagesBuffer.length > 10000) { + console.log("huge reset"); + this.resetBackendMessagesBuffer(); + } + return; + } + const s = this.backendMessagesBuffer.slice(5, i); + console.log("endup up with ", { s }); + this.resetBackendMessagesBuffer(); + logger.debug( + `handle_backend_message: parsing JSON payload ${JSON.stringify(s)}`, + ); + try { + const payload = JSON.parse(s); + this.publish({ cmd: "message", payload }); + } catch (err) { + logger.warn( + `handle_backend_message: error sending JSON payload ${JSON.stringify( + s, + )}, ${err}`, + ); + } + } + }; } function getCWD(pathHead, cwd?): string { From 9225889c50808d0c215a623495b1049d78fc97fd Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 23 Jan 2025 07:04:18 +0000 Subject: [PATCH 024/281] nats terminal: use a different subject for the commands --- docs/nats/devlog.md | 9 ++- .../nats-terminal-connection.ts | 58 +++++++++++-------- src/packages/project/nats/terminal.ts | 15 ++++- 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 19cba42f3f..b73acd9343 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -222,9 +222,14 @@ NOTE!!! The above consumer is ephemeral -- it disappears if we don't grab it via - [x] project should set up the stream for capturing terminal outputs. - [x] delete old messages with a given subject. `nats stream purge project-81e0c408-ac65-4114-bad5-5f4b6539bd0e-terminal --seq=7000` - there is a setting max\_msgs\_per\_subject on a stream, so **we just set that and are done!** Gees. It is too easy. -- [ ] handle the other messages like resize -- [ ] need to move those other messages to a different subject that isn't part of the stream!! +- [x] handle the other messages like resize +- [x] need to move those other messages to a different subject that isn't part of the stream!! - [ ] permissions for jetstream usage and access +- [ ] use non\-json for the data.... +- [ ] refactor code so basic parameters \(e.g., subject names, etc.\) are defined in one place that can be imported in both the frontend and backend. +- [ ] font size keyboard shortcut +- [ ] need a better algorithm for sizing since we don't know when a user disconnects! + - when one user proposes a size, all other clients get asked their current size and only those that respond matter. how to do this? ## [ ] Goal: Terminal and **compute server** diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index d67e21c071..407932cef1 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -12,6 +12,7 @@ export class NatsTerminalConnection extends EventEmitter { private project_id: string; private path: string; private subject: string; + private cmd_subject: string; private state: null | "running" | "init" | "closed"; private consumer?; // keep = optional number of messages to retain between clients/sessions/view, i.e., @@ -45,6 +46,7 @@ export class NatsTerminalConnection extends EventEmitter { this.closePaths = closePaths; // move to util so guaranteed in sync with project this.subject = `project.${project_id}.terminal.${sha1(path)}`; + this.cmd_subject = `project.${project_id}.terminal-cmd.${sha1(path)}`; } write = async (data) => { @@ -57,7 +59,6 @@ export class NatsTerminalConnection extends EventEmitter { await this.start(); } if (typeof data != "string") { - console.log("to project", data); if (data.cmd == "size") { const { rows, cols } = data; if ( @@ -72,12 +73,11 @@ export class NatsTerminalConnection extends EventEmitter { return; } } - const resp = await webapp_client.nats_client.project({ + await webapp_client.nats_client.project({ project_id: this.project_id, endpoint: "terminal-command", params: { path: this.path, ...data, client }, }); - console.log("got back ", resp); return; } const f = async () => { @@ -129,7 +129,8 @@ export class NatsTerminalConnection extends EventEmitter { this.state = "init"; await this.start(); this.consumer = await this.getConsumer(); - this.run(); + this.consumeDataStream(); + this.subscribeToCommands(); }; private handle = (mesg) => { @@ -139,29 +140,40 @@ export class NatsTerminalConnection extends EventEmitter { const x = jc.decode(mesg.data) as any; if (x?.data != null) { this.emit("data", x?.data); - } else { - switch (x.cmd) { - case "size": - this.terminalResize(x); - return; - case "message": - if (this.state != "running") { - return; - } - if (x.payload?.event == "open") { - this.openPaths(x.payload.paths); - } else if (x.payload?.event == "close") { - this.closePaths(x.payload.paths); - } - return; - default: - console.log("TODO -- unhandled message from project:", x); - return; + } + }; + + private subscribeToCommands = async () => { + const nc = await webapp_client.nats_client.getConnection(); + const sub = nc.subscribe(this.cmd_subject); + for await (const mesg of sub) { + if (this.state == "closed") { + return; } + this.handleCommand(mesg); + } + }; + + private handleCommand = async (mesg) => { + const x = jc.decode(mesg.data) as any; + switch (x.cmd) { + case "size": + this.terminalResize(x); + return; + case "message": + if (x.payload?.event == "open") { + this.openPaths(x.payload.paths); + } else if (x.payload?.event == "close") { + this.closePaths(x.payload.paths); + } + return; + default: + console.log("TODO -- unhandled message from project:", x); + return; } }; - private run = async () => { + private consumeDataStream = async () => { if (this.consumer == null) { return; } diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index ce5488cc02..5b207b9f06 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -94,6 +94,7 @@ class Session { private size?: { rows: number; cols: number }; // the subject where we publish our output public subject: string; + private cmd_subject: string; private state: "running" | "off" = "off"; private streamName: string; private keep: number; @@ -108,6 +109,7 @@ class Session { Math.min(this.options.keep ?? DEFAULT_KEEP, MAX_KEEP), ); this.subject = `project.${project_id}.terminal.${sha1(path)}`; + this.cmd_subject = `project.${project_id}.terminal-cmd.${sha1(path)}`; this.streamName = `project-${project_id}-terminal`; } @@ -206,6 +208,10 @@ class Session { this.nc.publish(this.subject, jc.encode(mesg)); }; + private publishCommand = (mesg) => { + this.nc.publish(this.cmd_subject, jc.encode(mesg)); + }; + private clientSizes = {}; setSize = ({ client, @@ -216,7 +222,10 @@ class Session { rows: number; cols: number; }) => { - this.clientSizes[client] = { rows, cols }; + //this.clientSizes[client] = { rows, cols }; + // just doing this silly hack for now -- we need to redo this algorithm to instead + // query all clients and when relevant, since no notion of connection. + this.clientSizes = { [client]: { rows, cols } }; this.resize(); }; @@ -234,7 +243,7 @@ class Session { try { this.setSizePty({ rows, cols }); // broadcast out new size - this.nc.publish(this.subject, jc.encode({ cmd: "size", rows, cols })); + this.publishCommand({ cmd: "size", rows, cols }); } catch (err) { logger.debug("terminal channel -- WARNING: unable to resize term", err); } @@ -324,7 +333,7 @@ class Session { ); try { const payload = JSON.parse(s); - this.publish({ cmd: "message", payload }); + this.publishCommand({ cmd: "message", payload }); } catch (err) { logger.warn( `handle_backend_message: error sending JSON payload ${JSON.stringify( From 2f3068ca9f1aa26c238f46adbca78c4bf8e1a5dc Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 23 Jan 2025 21:54:07 +0000 Subject: [PATCH 025/281] nats terminal: use pending field of mesg to better know when terminal history is loaded --- docs/nats/devlog.md | 11 ++++++++ .../terminal-editor/connected-terminal.ts | 4 +-- .../nats-terminal-connection.ts | 27 ++++++++++++++----- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index b73acd9343..9e9fb42e92 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -231,6 +231,17 @@ NOTE!!! The above consumer is ephemeral -- it disappears if we don't grab it via - [ ] need a better algorithm for sizing since we don't know when a user disconnects! - when one user proposes a size, all other clients get asked their current size and only those that respond matter. how to do this? +## [ ] Goal: Basic Collab Document Editing + +Plan. + +- Use a kv store hosted on nats to trac syncstring objects as before. This means anybody can participate \(browser, compute server, project\) without any need to contact the database, hence eliminating all proxying! +- Subject For Particular File: `project.${project_id}.patches.${sha1(path)}` +- Stream: Records everything with this subject `project.${project_id}.patches` +- It would be very nice if we can use the server assigned timestamps. +- For transitioning and de\-archiving, there must be a way to do this, since they have a backup/restore process +- + ## [ ] Goal: Terminal and **compute server** Another thing to do for compute servers: diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index f9978d864e..644c667dd9 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -379,7 +379,7 @@ export class Terminal { this.conn.write(data); } - private _handle_data_from_project(data: any): void { + private _handle_data_from_project = (data: any): void => { //console.log("data", data); this.assert_not_closed(); if (data == null) { @@ -402,7 +402,7 @@ export class Terminal { default: console.warn("TERMINAL: no way to handle data -- ", data); } - } + }; private activity() { this.project_actions.flag_file_activity(this.path); diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 407932cef1..c382e698f9 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -4,6 +4,7 @@ import { JSONCodec } from "nats.ws"; import sha1 from "sha1"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { uuid } from "@cocalc/util/misc"; +import { delay } from "awaiting"; const jc = JSONCodec(); const client = uuid(); @@ -178,23 +179,37 @@ export class NatsTerminalConnection extends EventEmitter { return; } const messages = await this.consumer.fetch({ - max_messages: 10000, + max_messages: 100000, // should only be a few hundred in practice expires: 1000, }); for await (const mesg of messages) { if (this.handle(mesg)) { return; } + if (mesg.info.pending == 0) { + // no further messages pending, so switch to consuming below + // TODO: I don't know if there is some chance to miss a message? + // This is a *terminal* so purely visual so not too critical. + break; + } } - if (this.state == "init") { - this.state = "running"; - this.emit("ready"); - } - // TODO: this loop runs until state = closed or this.consumer.closed()... ? + + this.setReady(); + for await (const mesg of await this.consumer.consume()) { if (this.handle(mesg)) { return; } } }; + + private setReady = async () => { + // wait until after render loop of terminal before allowing writing, + // or we get corruption. + await delay(100); // todo is there a better way to know how long to wait? + if (this.state == "init") { + this.state = "running"; + this.emit("ready"); + } + }; } From 6a14bf826b657212c308c8941fd1381b0222f65e Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 24 Jan 2025 01:08:12 +0000 Subject: [PATCH 026/281] nats: syncstrings first POC --- docs/nats/devlog.md | 10 +++++----- src/packages/frontend/client/nats.ts | 16 +++++++++++++++- src/packages/frontend/package.json | 1 + src/packages/pnpm-lock.yaml | 17 +++++++++++++++++ src/packages/project/nats/connection.ts | 19 +++++++++++++++++++ src/packages/project/nats/index.ts | 14 ++++---------- src/packages/project/nats/syncstrings.ts | 19 +++++++++++++++++++ src/packages/project/package.json | 1 + src/packages/server/nats/auth.ts | 6 +++++- src/packages/util/package.json | 18 +++++------------- 10 files changed, 91 insertions(+), 30 deletions(-) create mode 100644 src/packages/project/nats/connection.ts create mode 100644 src/packages/project/nats/syncstrings.ts diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 9e9fb42e92..01047976da 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -235,11 +235,11 @@ NOTE!!! The above consumer is ephemeral -- it disappears if we don't grab it via Plan. -- Use a kv store hosted on nats to trac syncstring objects as before. This means anybody can participate \(browser, compute server, project\) without any need to contact the database, hence eliminating all proxying! -- Subject For Particular File: `project.${project_id}.patches.${sha1(path)}` -- Stream: Records everything with this subject `project.${project_id}.patches` -- It would be very nice if we can use the server assigned timestamps. -- For transitioning and de\-archiving, there must be a way to do this, since they have a backup/restore process +- [ ] #now Use a kv store hosted on nats to trac syncstring objects as before. This means anybody can participate \(browser, compute server, project\) without any need to contact the database, hence eliminating all proxying! +- [ ] Subject For Particular File: `project.${project_id}.patches.${sha1(path)}` +- [ ] Stream: Records everything with this subject `project.${project_id}.patches` +- [ ] It would be very nice if we can use the server assigned timestamps. +- [ ] For transitioning and de\-archiving, there must be a way to do this, since they have a backup/restore process - ## [ ] Goal: Terminal and **compute server** diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 883790246c..96c979ee42 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -5,6 +5,8 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { join } from "path"; import { redux } from "../app-framework"; import * as jetstream from "@nats-io/jetstream"; +import { SyncStrings } from "@cocalc/util/nats/syncstrings"; +import sha1 from "sha1"; export class NatsClient { /*private*/ client: WebappClient; @@ -103,7 +105,19 @@ export class NatsClient { }; consumer = async (stream: string) => { - const js = jetstream.jetstream(await await this.getConnection()); + const js = jetstream.jetstream(await this.getConnection()); return await js.consumers.get(stream); }; + + // syncstrings in a given project + syncstrings = async (project_id: string) => { + const s = new SyncStrings({ + sha1, + jc: this.jc, + nc: await this.getConnection(), + project_id, + }); + await s.init(); + return s; + }; } diff --git a/src/packages/frontend/package.json b/src/packages/frontend/package.json index e211818272..1996a66bb0 100644 --- a/src/packages/frontend/package.json +++ b/src/packages/frontend/package.json @@ -58,6 +58,7 @@ "@lumino/widgets": "^1.31.1", "@microlink/react-json-view": "^1.23.3", "@nats-io/jetstream": "3.0.0-36", + "@nats-io/kv": "3.0.0-30", "@orama/orama": "3.0.0-rc-3", "@react-hook/mouse-position": "^4.1.3", "@rinsuki/lz4-ts": "^1.0.1", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 4e5d2f41e1..d702ee8056 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -304,6 +304,9 @@ importers: '@nats-io/jetstream': specifier: 3.0.0-36 version: 3.0.0-36 + '@nats-io/kv': + specifier: 3.0.0-30 + version: 3.0.0-30 '@orama/orama': specifier: 3.0.0-rc-3 version: 3.0.0-rc-3 @@ -1229,6 +1232,9 @@ importers: '@nats-io/jetstream': specifier: 3.0.0-36 version: 3.0.0-36 + '@nats-io/kv': + specifier: 3.0.0-30 + version: 3.0.0-30 '@nteract/messaging': specifier: ^7.0.20 version: 7.0.20 @@ -2007,6 +2013,9 @@ importers: '@isaacs/ttlcache': specifier: ^1.2.1 version: 1.4.1 + '@nats-io/kv': + specifier: 3.0.0-30 + version: 3.0.0-30 '@types/debug': specifier: ^4.1.12 version: 4.1.12 @@ -3857,6 +3866,9 @@ packages: '@nats-io/jetstream@3.0.0-36': resolution: {integrity: sha512-95+ftM+lXcUEwCl3ctzpJZCXiBUwRuSUUWmd4X8jJlNVZc/LUHH+3cxX+nC42sLHD+9zvIaMKPsyI0Ltbp+BSg==} + '@nats-io/kv@3.0.0-30': + resolution: {integrity: sha512-rg3y1/v4gTP6PATpAOK32cXgPfez8PZqC9r39vqT7vVIGUNKERFcX8JdqC+50B97ovqvlTSvItR9IsRNdYAIZg==} + '@nats-io/nats-core@3.0.0-49': resolution: {integrity: sha512-Xe7LjCdhtL4pXk2czwUE8Y1elTy/zo3ZzpoIwOO+/uJPughEsSxCpqygPrDqWcOG2uWVB9G1wxjg8r0Y9StovQ==} @@ -14864,6 +14876,11 @@ snapshots: dependencies: '@nats-io/nats-core': 3.0.0-49 + '@nats-io/kv@3.0.0-30': + dependencies: + '@nats-io/jetstream': 3.0.0-36 + '@nats-io/nats-core': 3.0.0-49 + '@nats-io/nats-core@3.0.0-49': dependencies: '@nats-io/nkeys': 2.0.2 diff --git a/src/packages/project/nats/connection.ts b/src/packages/project/nats/connection.ts new file mode 100644 index 0000000000..e415440338 --- /dev/null +++ b/src/packages/project/nats/connection.ts @@ -0,0 +1,19 @@ +import { getLogger } from "@cocalc/project/logger"; +import { connect, jwtAuthenticator } from "nats"; + +const logger = getLogger("project:nats:connection"); + +let nc: Awaited> | null = null; +export default async function getConnection() { + if (nc == null) { + logger.debug("initializing nats cocalc project connection"); + if (!process.env.COCALC_NATS_JWT) { + throw Error("environment variable COCALC_NATS_JWT *must* be set"); + } + nc = await connect({ + authenticator: jwtAuthenticator(process.env.COCALC_NATS_JWT), + }); + logger.debug(`connected to ${nc.getServer()}`); + } + return nc!; +} diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts index 8f2c1dfdb7..3de6213f95 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/nats/index.ts @@ -18,22 +18,16 @@ How to do development (so in a dev project doing cc-in-cc dev). */ import { getLogger } from "@cocalc/project/logger"; -import { connect, JSONCodec, jwtAuthenticator } from "nats"; +import { JSONCodec } from "nats"; import { project_id } from "@cocalc/project/data"; import { handleExecShellCode } from "@cocalc/project/exec_shell_code"; import { realpath } from "@cocalc/project/browser-websocket/realpath"; +import getConnection from "./connection"; -const logger = getLogger("server:nats"); +const logger = getLogger("project:nats"); export default async function initNatsServer() { - logger.debug("initializing nats cocalc project server"); - if (!process.env.COCALC_NATS_JWT) { - throw Error("environment variable COCALC_NATS_JWT *must* be set"); - } - const nc = await connect({ - authenticator: jwtAuthenticator(process.env.COCALC_NATS_JWT), - }); - logger.debug(`connected to ${nc.getServer()}`); + const nc = await getConnection(); initAPI(nc); } diff --git a/src/packages/project/nats/syncstrings.ts b/src/packages/project/nats/syncstrings.ts new file mode 100644 index 0000000000..526c22050b --- /dev/null +++ b/src/packages/project/nats/syncstrings.ts @@ -0,0 +1,19 @@ +import getConnection from "./connection"; +import { project_id } from "@cocalc/project/data"; +import { JSONCodec } from "nats"; +import { sha1 } from "@cocalc/backend/sha1"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { SyncStrings } from "@cocalc/util/nats/syncstrings"; + +const jc = JSONCodec(); + +let _syncstrings: null | SyncStrings; +const syncstrings = reuseInFlight(async () => { + if (_syncstrings == null) { + const nc = await getConnection(); + _syncstrings = new SyncStrings({ sha1, jc, nc, project_id }); + await _syncstrings.init(); + } + return _syncstrings!; +}); +export default syncstrings; diff --git a/src/packages/project/package.json b/src/packages/project/package.json index 81dd538103..6f9ac308f7 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -32,6 +32,7 @@ "@cocalc/terminal": "workspace:*", "@cocalc/util": "workspace:*", "@nats-io/jetstream": "3.0.0-36", + "@nats-io/kv": "3.0.0-30", "@nteract/messaging": "^7.0.20", "@types/lodash": "^4.14.202", "@types/primus": "^7.3.9", diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 4a248407c6..6eb7f37ec6 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -121,6 +121,8 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { for (const { project_id, group } of rows) { goalPub.add(`project.${project_id}.api.${group}.${userId}`); goalSub.add(`project.${project_id}.>`); + goalPub.add(`$KV.syncstrings.${project_id}.>`); + goalSub.add(`$KV.syncstrings.${project_id}.>`); } // TODO: there will be other subjects // TODO: something similar for projects, e.g., they can publish to a channel that browser clients @@ -129,8 +131,10 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { // the project can publish to anything under its own subject: goalPub.add(`project.${userId}.>`); goalSub.add(`project.${userId}.>`); + goalPub.add(`$KV.syncstrings.${userId}.>`); + goalSub.add(`$KV.syncstrings.${userId}.>`); } - // TEMPORARY: for learsning jetstream! + // TEMPORARY: for learning jetstream! goalPub.add("$JS.>"); goalSub.add("$JS.>"); diff --git a/src/packages/util/package.json b/src/packages/util/package.json index aadc9db6f4..34c0a8a00a 100644 --- a/src/packages/util/package.json +++ b/src/packages/util/package.json @@ -12,7 +12,8 @@ "./sync/table": "./dist/sync/table/index.js", "./sync/editor/db": "./dist/sync/editor/db/index.js", "./licenses/purchase/*": "./dist/licenses/purchase/*.js", - "./redux/*": "./dist/redux/*.js" + "./redux/*": "./dist/redux/*.js", + "./nats/*": "./dist/nats/*.js" }, "scripts": { "preinstall": "npx only-allow pnpm", @@ -21,25 +22,16 @@ "test": "pnpm exec jest", "prepublishOnly": "pnpm test" }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "mathjax", - "markdown", - "cocalc" - ], + "keywords": ["utilities", "mathjax", "markdown", "cocalc"], "license": "SEE LICENSE.md", "dependencies-COMMENT": "We must install react so that react-intl doesn't install the rwrong version! See https://github.com/sagemathinc/cocalc/issues/8132", "dependencies": { "@ant-design/colors": "^6.0.0", "@cocalc/util": "workspace:*", "@isaacs/ttlcache": "^1.2.1", + "@nats-io/kv": "3.0.0-30", "@types/debug": "^4.1.12", "async": "^1.5.2", "awaiting": "^3.0.0", From 244eebbffb57207b721f87ec26e41e6725b78a2d Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 24 Jan 2025 05:00:25 +0000 Subject: [PATCH 027/281] nats: synctable --- src/packages/frontend/client/nats.ts | 24 +++-- src/packages/project/nats/syncstrings.ts | 19 ---- src/packages/project/nats/synctable.ts | 27 +++++ src/packages/util/nats/synctable.ts | 130 +++++++++++++++++++++++ 4 files changed, 173 insertions(+), 27 deletions(-) delete mode 100644 src/packages/project/nats/syncstrings.ts create mode 100644 src/packages/project/nats/synctable.ts create mode 100644 src/packages/util/nats/synctable.ts diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 96c979ee42..d3ab41434f 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -5,8 +5,10 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { join } from "path"; import { redux } from "../app-framework"; import * as jetstream from "@nats-io/jetstream"; -import { SyncStrings } from "@cocalc/util/nats/syncstrings"; +import { SyncTable } from "@cocalc/util/nats/synctable"; +import { parse_query } from "@cocalc/sync/table/util"; import sha1 from "sha1"; +import { keys } from "lodash"; export class NatsClient { /*private*/ client: WebappClient; @@ -109,13 +111,19 @@ export class NatsClient { return await js.consumers.get(stream); }; - // syncstrings in a given project - syncstrings = async (project_id: string) => { - const s = new SyncStrings({ - sha1, - jc: this.jc, - nc: await this.getConnection(), - project_id, + synctable = async (query, project_id?) => { + query = parse_query(query); + if (project_id) { + const table = keys(query)[0]; + query[table][0].project_id = project_id; + } + const s = new SyncTable({ + query, + env: { + sha1, + jc: this.jc, + nc: await this.getConnection(), + }, }); await s.init(); return s; diff --git a/src/packages/project/nats/syncstrings.ts b/src/packages/project/nats/syncstrings.ts deleted file mode 100644 index 526c22050b..0000000000 --- a/src/packages/project/nats/syncstrings.ts +++ /dev/null @@ -1,19 +0,0 @@ -import getConnection from "./connection"; -import { project_id } from "@cocalc/project/data"; -import { JSONCodec } from "nats"; -import { sha1 } from "@cocalc/backend/sha1"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { SyncStrings } from "@cocalc/util/nats/syncstrings"; - -const jc = JSONCodec(); - -let _syncstrings: null | SyncStrings; -const syncstrings = reuseInFlight(async () => { - if (_syncstrings == null) { - const nc = await getConnection(); - _syncstrings = new SyncStrings({ sha1, jc, nc, project_id }); - await _syncstrings.init(); - } - return _syncstrings!; -}); -export default syncstrings; diff --git a/src/packages/project/nats/synctable.ts b/src/packages/project/nats/synctable.ts new file mode 100644 index 0000000000..ae0e69ff2d --- /dev/null +++ b/src/packages/project/nats/synctable.ts @@ -0,0 +1,27 @@ +import getConnection from "./connection"; +import { project_id } from "@cocalc/project/data"; +import { JSONCodec } from "nats"; +import { sha1 } from "@cocalc/backend/sha1"; +import { SyncTable } from "@cocalc/util/nats/synctable"; +import { parse_query } from "@cocalc/sync/table/util"; +import { keys } from "lodash"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; + +const jc = JSONCodec(); + +const cache: { [key]: SyncTable } = {}; +const synctable = reuseInFlight(async (query) => { + const key = JSON.stringify(query); + if (cache[key] == null) { + const nc = await getConnection(); + query = parse_query(query); + const table = keys(query)[0]; + query[table][0].project_id = project_id; + const s = new SyncTable({ query, env: { sha1, jc, nc } }); + await s.init(); + cache[key] = s; + } + return cache[key]; +}); + +export default synctable; diff --git a/src/packages/util/nats/synctable.ts b/src/packages/util/nats/synctable.ts new file mode 100644 index 0000000000..705f9c8009 --- /dev/null +++ b/src/packages/util/nats/synctable.ts @@ -0,0 +1,130 @@ +/* +Nats implementation of the idea of a "SyncTable". +*/ + +import { Kvm } from "@nats-io/kv"; +import sha1 from "sha1"; +import jsonStableStringify from "json-stable-stringify"; +import { keys } from "lodash"; +import { isValidUUID } from "@cocalc/util/misc"; +import { client_db } from "@cocalc/util/db-schema/client-db"; + +export async function getKv({ nc, table }) { + const kvm = new Kvm(nc); + return await kvm.create(table, { compression: true }); +} + +interface NatsEnv { + nc; // nats connection + jc; // jsoncodec + // compute sha1 hash efficiently (set differently on backend) + sha1?: (string) => string; +} + +function toKey(x): string | undefined { + if (x === undefined) { + return undefined; + } else if (typeof x === "object") { + return jsonStableStringify(x); + } else { + return `${x}`; + } +} + +export class SyncTable { + private kv?; + private nc; + private jc; + private sha1; + private table; + private primaryKeys: string[]; + private primaryKeysSet: Set; + private fields: string[]; + private project_id: string; + + constructor({ query, env }: { query; env: NatsEnv }) { + this.sha1 = env.sha1 ?? sha1; + this.nc = env.nc; + this.jc = env.jc; + const table = keys(query)[0]; + this.table = table; + this.project_id = query[table][0].project_id; + if (!isValidUUID(this.project_id)) { + throw Error("query MUST specify a valid project_id"); + } + this.primaryKeys = client_db.primary_keys(table); + this.primaryKeysSet = new Set(this.primaryKeys); + this.fields = keys(query[table][0]).filter( + (field) => !this.primaryKeysSet.has(field), + ); + } + + init = async () => { + this.kv = await getKv({ nc: this.nc, table: this.table }); + }; + + private natObjectKey = (obj): string => { + if (obj == null) { + throw Error("obj must be an object (not null)"); + } + // Function this.to_key to extract primary key from object + if (this.primaryKeys.length === 1) { + return this.sha1(toKey(obj[this.primaryKeys[0]] ?? "")); + } else { + // compound primary key + return this.sha1(toKey(this.primaryKeys.map((pk) => obj[pk]))); + } + }; + + private getKey = (obj, field?: string): string => { + const x = `${this.project_id}.${this.natObjectKey(obj)}`; + if (field == null) { + return x; + } else { + return `${x}.${field}`; + } + }; + + set = async (obj) => { + const key = this.getKey(obj); + for (const field in obj) { + if (!this.primaryKeysSet.has(field)) { + const value = this.jc.encode(obj[field]); + await this.kv.put(`${key}.${field}`, value); + } + } + }; + + get = async (obj, field?) => { + if (field == null) { + const s = { ...obj }; + const key = this.getKey(obj); + let nontrivial = false; + for (const field of this.fields) { + const mesg = await this.kv.get(`${key}.${field}`); + const val = mesg?.sm?.data ? this.jc.decode(mesg.sm.data) : null; + if (val != null) { + s[field] = val; + nontrivial = true; + } + } + return nontrivial ? s : undefined; + } + const mesg = await this.kv.get(this.getKey(obj, field)); + if (mesg == null) { + return undefined; + } + return this.jc.decode(mesg.sm.data); + }; + + // watch for changes in ONE object + async *watchOne(obj) { + const w = await this.kv.watch({ + key: this.getKey(this.getKey(obj), "*"), + }); + for await (const { key, value } of w) { + const field = key.slice(key.lastIndexOf(".") + 1); + yield { [field]: this.jc.decode(value) }; + } + } +} From a5787cc1a7d8f93d54be38d5026772fafe3d449a Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 24 Jan 2025 05:34:01 +0000 Subject: [PATCH 028/281] nats synctable -- get all --- docs/nats/devlog.md | 6 ++-- src/packages/util/nats/synctable.ts | 49 ++++++++++++++++++++++------- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 01047976da..6b3de45c44 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -235,11 +235,11 @@ NOTE!!! The above consumer is ephemeral -- it disappears if we don't grab it via Plan. -- [ ] #now Use a kv store hosted on nats to trac syncstring objects as before. This means anybody can participate \(browser, compute server, project\) without any need to contact the database, hence eliminating all proxying! +- [x] Use a kv store hosted on nats to trac syncstring objects as before. This means anybody can participate \(browser, compute server, project\) without any need to contact the database, hence eliminating all proxying! - [ ] Subject For Particular File: `project.${project_id}.patches.${sha1(path)}` - [ ] Stream: Records everything with this subject `project.${project_id}.patches` -- [ ] It would be very nice if we can use the server assigned timestamps. -- [ ] For transitioning and de\-archiving, there must be a way to do this, since they have a backup/restore process +- [ ] It would be very nice if we can use the server assigned timestamps.... but probably not + - [ ] For transitioning and de\-archiving, there must be a way to do this, since they have a backup/restore process - ## [ ] Goal: Terminal and **compute server** diff --git a/src/packages/util/nats/synctable.ts b/src/packages/util/nats/synctable.ts index 705f9c8009..9995c518bc 100644 --- a/src/packages/util/nats/synctable.ts +++ b/src/packages/util/nats/synctable.ts @@ -63,17 +63,20 @@ export class SyncTable { this.kv = await getKv({ nc: this.nc, table: this.table }); }; - private natObjectKey = (obj): string => { - if (obj == null) { - throw Error("obj must be an object (not null)"); - } - // Function this.to_key to extract primary key from object + private primaryString = (obj): string => { if (this.primaryKeys.length === 1) { - return this.sha1(toKey(obj[this.primaryKeys[0]] ?? "")); + return toKey(obj[this.primaryKeys[0]] ?? "")!; } else { // compound primary key - return this.sha1(toKey(this.primaryKeys.map((pk) => obj[pk]))); + return toKey(this.primaryKeys.map((pk) => obj[pk]))!; + } + }; + + private natObjectKey = (obj): string => { + if (obj == null) { + throw Error("obj must be an object (not null)"); } + return this.sha1(this.primaryString(obj)); }; private getKey = (obj, field?: string): string => { @@ -88,14 +91,36 @@ export class SyncTable { set = async (obj) => { const key = this.getKey(obj); for (const field in obj) { - if (!this.primaryKeysSet.has(field)) { - const value = this.jc.encode(obj[field]); - await this.kv.put(`${key}.${field}`, value); - } + const value = this.jc.encode(obj[field]); + await this.kv.put(`${key}.${field}`, value); } }; - get = async (obj, field?) => { + get = async (obj?, field?) => { + if (obj == null) { + // everything known in this table by the project + const keys = await this.kv.keys(`${this.project_id}.>`); + const all: any = {}; + for await (const key of keys) { + const mesg = await this.kv.get(key); + const val = mesg?.sm?.data ? this.jc.decode(mesg.sm.data) : null; + if (val != null) { + const i = key.lastIndexOf("."); + const field = key.slice(i + 1); + const prefix = key.slice(0, i); + if (all[prefix] == null) { + all[prefix] = {}; + } + const s = all[prefix]; + s[field] = val; + } + } + const final: any = {}; + for (const k in all) { + final[this.primaryString(all[k])] = all[k]; + } + return final; + } if (field == null) { const s = { ...obj }; const key = this.getKey(obj); From b1187b2f039e9f2cf5496182359b0c44dfed4286 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 24 Jan 2025 15:05:37 +0000 Subject: [PATCH 029/281] nats: add listings table --- docs/nats/devlog.md | 5 +++++ src/packages/server/nats/auth.ts | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 6b3de45c44..852629ad06 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -236,6 +236,11 @@ NOTE!!! The above consumer is ephemeral -- it disappears if we don't grab it via Plan. - [x] Use a kv store hosted on nats to trac syncstring objects as before. This means anybody can participate \(browser, compute server, project\) without any need to contact the database, hence eliminating all proxying! + +Next Goal \- collaborative file editing! This requires implementing the "ordered patches list" but on jetstream. Similar to the nats SyncTable I wrote yesterday, except will use jetstream directly, since it is an event stream, after all. + +--- + - [ ] Subject For Particular File: `project.${project_id}.patches.${sha1(path)}` - [ ] Stream: Records everything with this subject `project.${project_id}.patches` - [ ] It would be very nice if we can use the server assigned timestamps.... but probably not diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 6eb7f37ec6..53cb923aed 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -103,6 +103,8 @@ export async function getNatsUserJwt(cocalcUser: CoCalcUser): Promise { ).stdout.trim(); } +const PROJECT_TABLES = ["listings", "syncstrings"]; + export async function configureNatsUser(cocalcUser: CoCalcUser) { const name = getNatsUserName(cocalcUser); const userId = getCoCalcUserId(cocalcUser); @@ -121,8 +123,11 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { for (const { project_id, group } of rows) { goalPub.add(`project.${project_id}.api.${group}.${userId}`); goalSub.add(`project.${project_id}.>`); - goalPub.add(`$KV.syncstrings.${project_id}.>`); - goalSub.add(`$KV.syncstrings.${project_id}.>`); + + for (const table of PROJECT_TABLES) { + goalPub.add(`$KV.${table}.${project_id}.>`); + goalSub.add(`$KV.${table}.${project_id}.>`); + } } // TODO: there will be other subjects // TODO: something similar for projects, e.g., they can publish to a channel that browser clients @@ -131,8 +136,11 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { // the project can publish to anything under its own subject: goalPub.add(`project.${userId}.>`); goalSub.add(`project.${userId}.>`); - goalPub.add(`$KV.syncstrings.${userId}.>`); - goalSub.add(`$KV.syncstrings.${userId}.>`); + + for (const table of PROJECT_TABLES) { + goalPub.add(`$KV.${table}.${userId}.>`); + goalSub.add(`$KV.${table}.${userId}.>`); + } } // TEMPORARY: for learning jetstream! goalPub.add("$JS.>"); From 041e5bb75b4c89783e931fac0fecbc0bbbb1b90e Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 24 Jan 2025 15:54:54 +0000 Subject: [PATCH 030/281] nats: use single kv store for each project --- src/packages/server/nats/auth.ts | 16 ++++++---------- src/packages/util/nats/synctable.ts | 12 +++++++----- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 53cb923aed..29a27b6711 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -103,8 +103,6 @@ export async function getNatsUserJwt(cocalcUser: CoCalcUser): Promise { ).stdout.trim(); } -const PROJECT_TABLES = ["listings", "syncstrings"]; - export async function configureNatsUser(cocalcUser: CoCalcUser) { const name = getNatsUserName(cocalcUser); const userId = getCoCalcUserId(cocalcUser); @@ -124,10 +122,8 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { goalPub.add(`project.${project_id}.api.${group}.${userId}`); goalSub.add(`project.${project_id}.>`); - for (const table of PROJECT_TABLES) { - goalPub.add(`$KV.${table}.${project_id}.>`); - goalSub.add(`$KV.${table}.${project_id}.>`); - } + goalPub.add(`$KV.project-${project_id}.>`); + goalSub.add(`$KV.project-${project_id}.>`); } // TODO: there will be other subjects // TODO: something similar for projects, e.g., they can publish to a channel that browser clients @@ -137,10 +133,8 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { goalPub.add(`project.${userId}.>`); goalSub.add(`project.${userId}.>`); - for (const table of PROJECT_TABLES) { - goalPub.add(`$KV.${table}.${userId}.>`); - goalSub.add(`$KV.${table}.${userId}.>`); - } + goalPub.add(`$KV.project-${userId}.>`); + goalSub.add(`$KV.project-${userId}.>`); } // TEMPORARY: for learning jetstream! goalPub.add("$JS.>"); @@ -199,6 +193,8 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { changed = true; } if (sub.length > 0 || pub.length > 0) { + // TODO: I think there is --allow-pubsub which does both in one line, + // which would shorten this slightly if (sub.length > 0) { args.push("--allow-sub"); args.push(sub.join(",")); diff --git a/src/packages/util/nats/synctable.ts b/src/packages/util/nats/synctable.ts index 9995c518bc..4d2cdb794b 100644 --- a/src/packages/util/nats/synctable.ts +++ b/src/packages/util/nats/synctable.ts @@ -9,9 +9,9 @@ import { keys } from "lodash"; import { isValidUUID } from "@cocalc/util/misc"; import { client_db } from "@cocalc/util/db-schema/client-db"; -export async function getKv({ nc, table }) { +export async function getKv({ nc, project_id }) { const kvm = new Kvm(nc); - return await kvm.create(table, { compression: true }); + return await kvm.create(`project-${project_id}`, { compression: true }); } interface NatsEnv { @@ -60,7 +60,7 @@ export class SyncTable { } init = async () => { - this.kv = await getKv({ nc: this.nc, table: this.table }); + this.kv = await getKv({ nc: this.nc, project_id: this.project_id }); }; private primaryString = (obj): string => { @@ -80,7 +80,7 @@ export class SyncTable { }; private getKey = (obj, field?: string): string => { - const x = `${this.project_id}.${this.natObjectKey(obj)}`; + const x = `${this.table}.${this.natObjectKey(obj)}`; if (field == null) { return x; } else { @@ -99,7 +99,7 @@ export class SyncTable { get = async (obj?, field?) => { if (obj == null) { // everything known in this table by the project - const keys = await this.kv.keys(`${this.project_id}.>`); + const keys = await this.kv.keys(`${this.table}.>`); const all: any = {}; for await (const key of keys) { const mesg = await this.kv.get(key); @@ -125,6 +125,8 @@ export class SyncTable { const s = { ...obj }; const key = this.getKey(obj); let nontrivial = false; + // todo: possibly better to just ask for everything under ${key}.> + // and take what is needed? Not sure. for (const field of this.fields) { const mesg = await this.kv.get(`${key}.${field}`); const val = mesg?.sm?.data ? this.jc.decode(mesg.sm.data) : null; From 92e8d354a3be97358c596876f0d4ae3ca114f561 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 24 Jan 2025 15:58:54 +0000 Subject: [PATCH 031/281] reformat sorted-patch-list to use arrow functions --- .../sync/editor/generic/sorted-patch-list.ts | 86 +++++++++---------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/src/packages/sync/editor/generic/sorted-patch-list.ts b/src/packages/sync/editor/generic/sorted-patch-list.ts index a92d626f8e..1a7f56b411 100644 --- a/src/packages/sync/editor/generic/sorted-patch-list.ts +++ b/src/packages/sync/editor/generic/sorted-patch-list.ts @@ -37,10 +37,10 @@ export class SortedPatchList extends EventEmitter { this.from_str = from_str; } - public close(): void { + close = (): void => { this.removeAllListeners(); close(this); - } + }; /* Choose the next available time in ms that is congruent to m modulo n. The congruence condition is so that any time @@ -48,11 +48,11 @@ export class SortedPatchList extends EventEmitter { document with themselves -- two different users are guaranteed to not collide. Note: even if there is a collision, it will automatically fix itself very quickly. */ - public next_available_time( + next_available_time = ( time: Date | number, m: number = 0, - n: number = 1 - ): Date { + n: number = 1, + ): Date => { let t: number; if (typeof time === "number") { t = time; @@ -72,9 +72,9 @@ export class SortedPatchList extends EventEmitter { t += n; } return new Date(t); - } + }; - public add(patches: Patch[]): void { + add = (patches: Patch[]): void => { if (patches.length === 0) { // nothing to do return; @@ -125,9 +125,9 @@ export class SortedPatchList extends EventEmitter { this.patches = this.patches.concat(v); this.patches.sort(patch_cmp); } - } + }; - private newest_snapshot_time(): Date { + private newest_snapshot_time = (): Date => { let t0 = 0; let t: string; for (t in this.all_snapshot_times) { @@ -137,7 +137,7 @@ export class SortedPatchList extends EventEmitter { } } return new Date(t0); - } + }; /* value: Return the value of the document at the given (optional) @@ -155,7 +155,7 @@ export class SortedPatchList extends EventEmitter { as a building block to implement undo. We do not assume that without_times is sorted. */ - public value(time?: Date, force?: boolean, without_times?: Date[]): Document { + value = (time?: Date, force?: boolean, without_times?: Date[]): Document => { // oldest time that is skipped: let oldest_without_time: Date | undefined = undefined; // all skipped times. @@ -309,7 +309,7 @@ export class SortedPatchList extends EventEmitter { // files basically be massively broken! console.warn( "WARNING: unable to apply a patch -- skipping it", - err + err, ); } } @@ -338,11 +338,11 @@ export class SortedPatchList extends EventEmitter { */ return value; - } + }; // VERY Slow -- only for consistency checking purposes and debugging. // If snapshots=false, don't use snapshots. - public value_no_cache(time?: Date, snapshots: boolean = true): Document { + value_no_cache = (time?: Date, snapshots: boolean = true): Document => { let value: Document = this.from_str(""); // default in case no snapshots let start: number = 0; const prev_cutoff: Date = this.newest_snapshot_time(); @@ -378,11 +378,11 @@ export class SortedPatchList extends EventEmitter { } } return value; - } + }; // For testing/debugging. Go through the complete patch history and // verify that all snapshots are correct (or not -- in which case say so). - public validate_snapshots(): void { + validate_snapshots = (): void => { if (this.patches.length === 0) { return; } @@ -410,46 +410,46 @@ export class SortedPatchList extends EventEmitter { } i += 1; } - } + }; // integer index of user who made the patch at given point in time. // Throws an exception if there is no patch at that point in time. - public user_id(time): number { + user_id = (time): number => { const x = this.patch(time); if (x == null) { throw Error(`no patch at ${time}`); } return x.user_id; - } + }; // Returns time when patch was sent out, or undefined. This is // ONLY set if the patch was sent at a significantly different // time than when it was created, e.g., due to it being offline. // Throws an exception if there is no patch at that point in time. - public time_sent(time): Date | undefined { + time_sent = (time): Date | undefined => { return this.patch(time).sent; - } + }; // Patch at a given point in time. // Throws an exception if there is no patch at that point in time. - public patch(time): Patch { + patch = (time): Patch => { const p = this.times[time.valueOf()]; if (p == null) { throw Error(`no patch at ${time}`); } return p; - } + }; - public versions(): Date[] { + versions = (): Date[] => { // Compute and cache result,then return it; result gets cleared when new patches added. if (this.versions_cache == null) { this.versions_cache = this.patches.map((x) => x.time); } return this.versions_cache; - } + }; // Show the history of this document; used mainly for debugging purposes. - public show_history({ + show_history = ({ milliseconds, trunc, log, @@ -457,7 +457,7 @@ export class SortedPatchList extends EventEmitter { milliseconds?: boolean; trunc?: number; log?: Function; - } = {}) { + } = {}) => { if (milliseconds === undefined) { milliseconds = false; } @@ -482,7 +482,7 @@ export class SortedPatchList extends EventEmitter { i, x.user_id, tm_show, - trunc_middle(JSON.stringify(x.patch), trunc) + trunc_middle(JSON.stringify(x.patch), trunc), ); if (s === undefined) { s = this.from_str(x.snapshot != null ? x.snapshot : ""); @@ -498,18 +498,18 @@ export class SortedPatchList extends EventEmitter { s = s.apply_patch(x.patch); } else { log( - `prev=${x.prev.valueOf()} is missing, so not applying this patch` + `prev=${x.prev.valueOf()} is missing, so not applying this patch`, ); } } first_time = false; log( x.snapshot ? "(SNAPSHOT) " : " ", - trunc_middle(s.to_str(), trunc).trim() + trunc_middle(s.to_str(), trunc).trim(), ); i += 1; } - } + }; /* This function does not MAKE a snapshot; it just returns the time at which we must plan to make a snapshot. @@ -537,10 +537,10 @@ export class SortedPatchList extends EventEmitter { more need to be made. This isn't maximally efficient, in that several extra snapshots might get made, but maybe that is OK. */ - public time_of_unmade_periodic_snapshot( + time_of_unmade_periodic_snapshot = ( interval: number, - max_size: number - ): Date | undefined { + max_size: number, + ): Date | undefined => { const n = this.patches.length - 1; let cur_size: number = 0; for (let i = n; i >= 0; i--) { @@ -580,11 +580,11 @@ export class SortedPatchList extends EventEmitter { } } } - } + }; // Times of all snapshots in memory on this client; these are // the only ones we need to worry about for offline patches... - public snapshot_times(): Date[] { + snapshot_times = (): Date[] => { const v: Date[] = []; let t: string; for (t in this.all_snapshot_times) { @@ -592,22 +592,22 @@ export class SortedPatchList extends EventEmitter { } v.sort(cmp_Date); return v; - } + }; /* Return the most recent time of a patch, or undefined if there are no patches. */ - public newest_patch_time(): Date | undefined { + newest_patch_time = (): Date | undefined => { if (this.patches.length === 0) { return; } return this.patches[this.patches.length - 1].time; - } + }; - public count(): number { + count = (): number => { return this.patches.length; - } + }; - public export(): Patch[] { + export = (): Patch[] => { return deep_copy(this.patches); - } + }; } From 6e327152b4dc90c5a9812b572d740e29afe93723 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 24 Jan 2025 17:26:15 +0000 Subject: [PATCH 032/281] nats patches stream -- work in progress --- docs/nats/devlog.md | 2 +- src/packages/frontend/client/nats.ts | 12 +- src/packages/pnpm-lock.yaml | 3 + src/packages/project/nats/synctable.ts | 20 ++- src/packages/project/nats/terminal.ts | 4 +- src/packages/sync/editor/generic/sync-doc.ts | 10 +- .../nats/{synctable.ts => synctable-kv.ts} | 8 +- src/packages/util/nats/synctable-stream.ts | 153 ++++++++++++++++++ src/packages/util/package.json | 15 +- 9 files changed, 208 insertions(+), 19 deletions(-) rename src/packages/util/nats/{synctable.ts => synctable-kv.ts} (95%) create mode 100644 src/packages/util/nats/synctable-stream.ts diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 852629ad06..bea9d5dd1e 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -237,7 +237,7 @@ Plan. - [x] Use a kv store hosted on nats to trac syncstring objects as before. This means anybody can participate \(browser, compute server, project\) without any need to contact the database, hence eliminating all proxying! -Next Goal \- collaborative file editing! This requires implementing the "ordered patches list" but on jetstream. Similar to the nats SyncTable I wrote yesterday, except will use jetstream directly, since it is an event stream, after all. +#now Next Goal \- collaborative file editing! This requires implementing the "ordered patches list" but on jetstream. Similar to the nats SyncTable I wrote yesterday, except will use jetstream directly, since it is an event stream, after all. --- diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index d3ab41434f..a73f81ae06 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -5,7 +5,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { join } from "path"; import { redux } from "../app-framework"; import * as jetstream from "@nats-io/jetstream"; -import { SyncTable } from "@cocalc/util/nats/synctable"; +import { SyncTableKV } from "@cocalc/util/nats/synctable-kv"; import { parse_query } from "@cocalc/sync/table/util"; import sha1 from "sha1"; import { keys } from "lodash"; @@ -111,13 +111,15 @@ export class NatsClient { return await js.consumers.get(stream); }; - synctable = async (query, project_id?) => { + synctable = async (query, obj?) => { query = parse_query(query); - if (project_id) { + if (obj != null) { const table = keys(query)[0]; - query[table][0].project_id = project_id; + for (const k in obj) { + query[table][0][k] = obj[k]; + } } - const s = new SyncTable({ + const s = new SyncTableKV({ query, env: { sha1, diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index d702ee8056..a7951b2bb5 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -2013,6 +2013,9 @@ importers: '@isaacs/ttlcache': specifier: ^1.2.1 version: 1.4.1 + '@nats-io/jetstream': + specifier: 3.0.0-36 + version: 3.0.0-36 '@nats-io/kv': specifier: 3.0.0-30 version: 3.0.0-30 diff --git a/src/packages/project/nats/synctable.ts b/src/packages/project/nats/synctable.ts index ae0e69ff2d..87781903a4 100644 --- a/src/packages/project/nats/synctable.ts +++ b/src/packages/project/nats/synctable.ts @@ -2,21 +2,28 @@ import getConnection from "./connection"; import { project_id } from "@cocalc/project/data"; import { JSONCodec } from "nats"; import { sha1 } from "@cocalc/backend/sha1"; -import { SyncTable } from "@cocalc/util/nats/synctable"; +import { SyncTableKV } from "@cocalc/util/nats/synctable-kv"; +import { SyncTableStream } from "@cocalc/util/nats/synctable-stream"; import { parse_query } from "@cocalc/sync/table/util"; import { keys } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; const jc = JSONCodec(); -const cache: { [key]: SyncTable } = {}; -const synctable = reuseInFlight(async (query) => { +const cache: { [key: string]: SyncTableKV | SyncTableStream } = {}; +const synctable = reuseInFlight(async (query, obj?) => { const key = JSON.stringify(query); if (cache[key] == null) { const nc = await getConnection(); query = parse_query(query); const table = keys(query)[0]; + if (obj != null) { + for (const k in obj) { + query[table][0][k] = obj[k]; + } + } query[table][0].project_id = project_id; + const SyncTable = getClass(table); const s = new SyncTable({ query, env: { sha1, jc, nc } }); await s.init(); cache[key] = s; @@ -25,3 +32,10 @@ const synctable = reuseInFlight(async (query) => { }); export default synctable; + +function getClass(table) { + if (table == "patches") { + return SyncTableStream; + } + return SyncTableKV; +} diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index 5b207b9f06..a8615f651a 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -147,7 +147,7 @@ class Session { return path; }; - getStream = async () => { + createStream = async () => { // idempotent so don't have to check if there is already a stream const nc = this.nc; const jsm = await jetstreamManager(nc); @@ -192,7 +192,7 @@ class Session { cols: this.size?.cols, }); this.state = "running"; - await this.getStream(); + await this.createStream(); this.pty.onData((data) => { this.handleBackendMessages(data); this.publish({ data }); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 7317bc1f96..4303746062 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1712,8 +1712,8 @@ export class SyncDoc extends EventEmitter { const doc = patch_list.value(); this.last = this.doc = doc; - this.patches_table.on("change", this.handle_patch_update.bind(this)); - this.patches_table.on("saved", this.handle_offline.bind(this)); + this.patches_table.on("change", this.handle_patch_update); + this.patches_table.on("saved", this.handle_offline); this.patch_list = patch_list; // this only potentially happens for tables in the project, @@ -2377,7 +2377,7 @@ export class SyncDoc extends EventEmitter { are relatively old; if so, we mark them as such and also possibly recompute snapshots. */ - private async handle_offline(data): Promise { + private handle_offline = async (data): Promise => { this.assert_not_closed("handle_offline"); const now: Date = this.client.server_time(); let oldest: Date | undefined = undefined; @@ -3143,7 +3143,7 @@ export class SyncDoc extends EventEmitter { It handles update of the remote version, updating our live version as a result. */ - private async handle_patch_update(changed_keys): Promise { + private handle_patch_update = async (changed_keys): Promise => { if (changed_keys == null || changed_keys.length === 0) { // this happens right now when we do a save. return; @@ -3162,7 +3162,7 @@ export class SyncDoc extends EventEmitter { await delay(1); await this.handle_patch_update_queue(); dbg("done"); - } + }; /* Whenever new patches are added to this.patches_table, diff --git a/src/packages/util/nats/synctable.ts b/src/packages/util/nats/synctable-kv.ts similarity index 95% rename from src/packages/util/nats/synctable.ts rename to src/packages/util/nats/synctable-kv.ts index 4d2cdb794b..c1012b58a1 100644 --- a/src/packages/util/nats/synctable.ts +++ b/src/packages/util/nats/synctable-kv.ts @@ -1,5 +1,11 @@ /* Nats implementation of the idea of a "SyncTable". + +This is ONLY for synctables in the scope of a single project, e.g., +syncstrings, listings, etc. + +It uses a SINGLE NATS key-value store to represent +*all* SyncTables in a single project. */ import { Kvm } from "@nats-io/kv"; @@ -31,7 +37,7 @@ function toKey(x): string | undefined { } } -export class SyncTable { +export class SyncTableKV { private kv?; private nc; private jc; diff --git a/src/packages/util/nats/synctable-stream.ts b/src/packages/util/nats/synctable-stream.ts new file mode 100644 index 0000000000..2c62addb40 --- /dev/null +++ b/src/packages/util/nats/synctable-stream.ts @@ -0,0 +1,153 @@ +/* +Nats implementation of the idea of a "SyncTable", but +for streaming data. + +This is ONLY for the scope of patches in a single project. + +It uses a NATS stream to store the elements in a well defined order. + + +*/ + +import { jetstreamManager, jetstream } from "@nats-io/jetstream"; +import sha1 from "sha1"; +import jsonStableStringify from "json-stable-stringify"; +import { keys } from "lodash"; +import { isValidUUID } from "@cocalc/util/misc"; +import { client_db } from "@cocalc/util/db-schema/client-db"; + +interface NatsEnv { + nc; // nats connection + jc; // jsoncodec + // compute sha1 hash efficiently (set differently on backend) + sha1?: (string) => string; +} + +function toKey(x): string | undefined { + if (x === undefined) { + return undefined; + } else if (typeof x === "object") { + return jsonStableStringify(x); + } else { + return `${x}`; + } +} + +export class SyncTableStream { + private kv?; + private nc; + private jc; + private sha1; + private table; + private primaryKeys: string[]; + private primaryKeysSet: Set; + private fields: string[]; + private project_id: string; + private streamName: string; + private path: string; + private subject: string; + private consumer?; + + constructor({ query, env }: { query; env: NatsEnv }) { + this.sha1 = env.sha1 ?? sha1; + this.nc = env.nc; + this.jc = env.jc; + const table = keys(query)[0]; + this.table = table; + this.project_id = query[table][0].project_id; + if (!isValidUUID(this.project_id)) { + throw Error("query MUST specify a valid project_id"); + } + this.path = query[table][0].path; + if (!this.path) { + throw Error("path MUST be specified"); + } + query[table][0].string_id = this.sha1(`${this.project_id}${this.path}`); + this.streamName = `${this.table}-${query[table][0].string_id}`; + this.subject = `project.${this.project_id}.${this.table}.${query[table][0].string_id}`; + this.primaryKeys = client_db.primary_keys(table); + this.primaryKeysSet = new Set(this.primaryKeys); + this.fields = keys(query[table][0]).filter( + (field) => !this.primaryKeysSet.has(field), + ); + } + + private createStream = async () => { + const jsm = await jetstreamManager(this.nc); + try { + await jsm.streams.add({ + name: this.streamName, + subjects: [this.subject], + compression: "s2", + }); + } catch (_err) { + // probably already exists + await jsm.streams.update(this.streamName, { + subjects: [this.subject], + compression: "s2" as any, + }); + } + }; + + private getConsumer = async () => { + const js = jetstream(this.nc); + const jsm = await jetstreamManager(this.nc); + // making an ephemeral consumer + const { name } = await jsm.consumers.add(this.streamName, { + filter_subject: this.subject, + }); + return await js.consumers.get(this.streamName, name); + }; + + init = async () => { + await this.createStream(); + this.consumer = await this.getConsumer(); + this.getMessages(); + }; + + private primaryString = (obj): string => {}; + + private natObjectKey = (obj): string => {}; + + private getKey = (obj, field?: string): string => {}; + + private publish = (mesg) => { + this.nc.publish(this.subject, this.jc.encode(mesg)); + }; + + set = async (obj) => { + delete obj["string_id"]; // redundant + this.publish(obj); + }; + + private handle = (mesg) => { + const x = this.jc.decode(mesg.data); + console.log(x); + return false; + }; + + private getMessages = async () => { + const consumer = this.consumer!; + const messages = await consumer.fetch({ + max_messages: 100000, + }); + for await (const mesg of messages) { + if (this.handle(mesg)) { + return; + } + if (mesg.info.pending == 0) { + // no further messages + break; + } + } + for await (const mesg of await consumer.consume()) { + if (this.handle(mesg)) { + return; + } + } + }; + + get = async (obj?, field?) => { + console.log("get: TODO"); + }; +} diff --git a/src/packages/util/package.json b/src/packages/util/package.json index 34c0a8a00a..5fbee209c9 100644 --- a/src/packages/util/package.json +++ b/src/packages/util/package.json @@ -22,15 +22,26 @@ "test": "pnpm exec jest", "prepublishOnly": "pnpm test" }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["utilities", "mathjax", "markdown", "cocalc"], + "keywords": [ + "utilities", + "mathjax", + "markdown", + "cocalc" + ], "license": "SEE LICENSE.md", "dependencies-COMMENT": "We must install react so that react-intl doesn't install the rwrong version! See https://github.com/sagemathinc/cocalc/issues/8132", "dependencies": { "@ant-design/colors": "^6.0.0", "@cocalc/util": "workspace:*", "@isaacs/ttlcache": "^1.2.1", + "@nats-io/jetstream": "3.0.0-36", "@nats-io/kv": "3.0.0-30", "@types/debug": "^4.1.12", "async": "^1.5.2", From f6901c0d994c673e7fd2c8cbea86ea6f8f2915a2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 25 Jan 2025 02:15:55 +0000 Subject: [PATCH 033/281] nats: quick broken proof of concept test of file editing using nats... --- src/packages/frontend/client/client.ts | 7 +- src/packages/frontend/client/nats.ts | 6 +- src/packages/project/client.ts | 7 +- src/packages/project/nats/synctable.ts | 15 +-- src/packages/server/nats/auth.ts | 6 +- src/packages/sync/editor/generic/sync-doc.ts | 90 ++++++++----- src/packages/sync/editor/generic/types.ts | 2 + .../sync/editor/string/test/client-test.ts | 4 + src/packages/util/nats/synctable-stream.ts | 121 ++++++++++++++---- src/packages/util/nats/synctable.ts | 13 ++ 10 files changed, 199 insertions(+), 72 deletions(-) create mode 100644 src/packages/util/nats/synctable.ts diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 5bd3e13ea1..3ba556a8fe 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -76,6 +76,7 @@ export interface WebappClient extends EventEmitter { get_username: Function; is_signed_in: () => boolean; synctable_project: Function; + synctable_nats: Function; project_websocket: Function; prettier: Function; exec: Function; @@ -157,6 +158,7 @@ class Client extends EventEmitter implements WebappClient { get_username: Function; is_signed_in: () => boolean; synctable_project: Function; + synctable_nats: Function; project_websocket: Function; prettier: Function; exec: Function; @@ -245,7 +247,9 @@ class Client extends EventEmitter implements WebappClient { this.idle_reset = this.idle_client.idle_reset.bind(this.idle_client); this.exec = this.project_client.exec.bind(this.project_client); - this.touch_project = this.project_client.touch_project.bind(this.project_client); + this.touch_project = this.project_client.touch_project.bind( + this.project_client, + ); this.ipywidgetsGetBuffer = this.project_client.ipywidgetsGetBuffer.bind( this.project_client, ); @@ -256,6 +260,7 @@ class Client extends EventEmitter implements WebappClient { this.synctable_project = this.sync_client.synctable_project.bind( this.sync_client, ); + this.synctable_nats = this.nats_client.synctable; this.query = this.query_client.query.bind(this.query_client); this.async_query = this.query_client.query.bind(this.query_client); diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index a73f81ae06..16726cc203 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -5,7 +5,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { join } from "path"; import { redux } from "../app-framework"; import * as jetstream from "@nats-io/jetstream"; -import { SyncTableKV } from "@cocalc/util/nats/synctable-kv"; +import { createSyncTable, type SyncTable } from "@cocalc/util/nats/synctable"; import { parse_query } from "@cocalc/sync/table/util"; import sha1 from "sha1"; import { keys } from "lodash"; @@ -111,7 +111,7 @@ export class NatsClient { return await js.consumers.get(stream); }; - synctable = async (query, obj?) => { + synctable = async (query, obj?): Promise => { query = parse_query(query); if (obj != null) { const table = keys(query)[0]; @@ -119,7 +119,7 @@ export class NatsClient { query[table][0][k] = obj[k]; } } - const s = new SyncTableKV({ + const s = createSyncTable({ query, env: { sha1, diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index a9f9c2ffd2..8364998649 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -48,6 +48,7 @@ import * as sage_session from "./sage_session"; import { getListingsTable } from "@cocalc/project/sync/listings"; import { get_synctable } from "./sync/open-synctables"; import { get_syncdoc } from "./sync/sync-doc"; +import synctable_nats from "@cocalc/project/nats/synctable"; const winston = getLogger("client"); @@ -500,6 +501,10 @@ export class Client extends EventEmitter implements ProjectClientInterface { return the_synctable; } + synctable_nats = async (query, obj?) => { + return await synctable_nats(query, obj); + }; + // WARNING: making two of the exact same sync_string or sync_db will definitely // lead to corruption! @@ -612,7 +617,7 @@ export class Client extends EventEmitter implements ProjectClientInterface { } // no-op; assumed async api - touch_project(_project_id: string, _compute_server_id?:number) {} + touch_project(_project_id: string, _compute_server_id?: number) {} async get_syncdoc_history(string_id: string, patches = false) { const dbg = this.dbg("get_syncdoc_history"); diff --git a/src/packages/project/nats/synctable.ts b/src/packages/project/nats/synctable.ts index 87781903a4..e27d6963b5 100644 --- a/src/packages/project/nats/synctable.ts +++ b/src/packages/project/nats/synctable.ts @@ -2,15 +2,14 @@ import getConnection from "./connection"; import { project_id } from "@cocalc/project/data"; import { JSONCodec } from "nats"; import { sha1 } from "@cocalc/backend/sha1"; -import { SyncTableKV } from "@cocalc/util/nats/synctable-kv"; -import { SyncTableStream } from "@cocalc/util/nats/synctable-stream"; +import { createSyncTable, type SyncTable } from "@cocalc/util/nats/synctable"; import { parse_query } from "@cocalc/sync/table/util"; import { keys } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; const jc = JSONCodec(); -const cache: { [key: string]: SyncTableKV | SyncTableStream } = {}; +const cache: { [key: string]: SyncTable } = {}; const synctable = reuseInFlight(async (query, obj?) => { const key = JSON.stringify(query); if (cache[key] == null) { @@ -23,8 +22,7 @@ const synctable = reuseInFlight(async (query, obj?) => { } } query[table][0].project_id = project_id; - const SyncTable = getClass(table); - const s = new SyncTable({ query, env: { sha1, jc, nc } }); + const s = createSyncTable({ query, env: { sha1, jc, nc } }); await s.init(); cache[key] = s; } @@ -32,10 +30,3 @@ const synctable = reuseInFlight(async (query, obj?) => { }); export default synctable; - -function getClass(table) { - if (table == "patches") { - return SyncTableStream; - } - return SyncTableKV; -} diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 29a27b6711..5a1fcdd46b 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -118,8 +118,10 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { // all RUNNING projects with the user's group const query = `SELECT project_id, users#>>'{${userId},group}' AS group FROM projects WHERE state#>>'{state}'='running' AND users ? '${userId}' ORDER BY project_id`; const { rows } = await pool.query(query); - for (const { project_id, group } of rows) { - goalPub.add(`project.${project_id}.api.${group}.${userId}`); + for (const { project_id /*, group */ } of rows) { + // TODO - unsure -- do we need proven identity *in* project? + //goalPub.add(`project.${project_id}.api.${group}.${userId}`); + goalPub.add(`project.${project_id}.>`); goalSub.add(`project.${project_id}.>`); goalPub.add(`$KV.project-${project_id}.>`); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 4303746062..b6c9f63c96 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1005,6 +1005,13 @@ export class SyncDoc extends EventEmitter { is sorted from oldest to newest. */ public versions = (): Date[] => { this.assert_table_is_ready("patches"); + + // check for new NATS function and use that if available. + const { getSortedTimes } = this.patches_table as any; + if (getSortedTimes != null) { + return getSortedTimes(); + } + const v: Date[] = []; const s: Map | undefined = this.patches_table.get(); if (s == null) { @@ -1235,25 +1242,32 @@ export class SyncDoc extends EventEmitter { options.push({ ephemeral: true }); } let synctable; - switch (this.data_server) { - case "project": - synctable = await this.client.synctable_project( - this.project_id, - query, - options, - throttle_changes, - this.id, - ); - break; - case "database": - synctable = await this.client.synctable_database( - query, - options, - throttle_changes, - ); - break; - default: - throw Error(`uknown server ${this.data_server}`); + if (this.path == "nats.txt" && query.patches) { + synctable = await this.client.synctable_nats(query, { + project_id: this.project_id, + path: this.path, + }); + } else { + switch (this.data_server) { + case "project": + synctable = await this.client.synctable_project( + this.project_id, + query, + options, + throttle_changes, + this.id, + ); + break; + case "database": + synctable = await this.client.synctable_database( + query, + options, + throttle_changes, + ); + break; + default: + throw Error(`uknown server ${this.data_server}`); + } } // We listen and log error events. This is useful because in some settings, e.g., // in the project, an eventemitter with no listener for errors, which has an error, @@ -1319,6 +1333,9 @@ export class SyncDoc extends EventEmitter { // Used for internal debug logging private dbg = (f: string = ""): Function => { + if (this.path == "nats.txt") { + return (...args) => console.log(f, ...args); + } return this.client?.dbg(`SyncDoc('${this.path}').${f}`); }; @@ -1402,6 +1419,9 @@ export class SyncDoc extends EventEmitter { // wait until the syncstring table is ready to be // used (so extracted from archive, etc.), private async wait_until_fully_ready(): Promise { + if (this.path == "nats.txt") { + return; + } this.assert_not_closed("wait_until_fully_ready"); const dbg = this.dbg("wait_until_fully_ready"); dbg(); @@ -1444,7 +1464,6 @@ export class SyncDoc extends EventEmitter { if (init.error) { throw Error(init.error); } - assertDefined(this.patch_list); if ( !this.client.is_project() && @@ -1682,11 +1701,8 @@ export class SyncDoc extends EventEmitter { const patch_list = new SortedPatchList(this._from_str); dbg("opening the table..."); - this.patches_table = await this.synctable( - { patches: [this.patch_table_query(this.last_snapshot)] }, - [], - this.patch_interval, - ); + const query = { patches: [this.patch_table_query(this.last_snapshot)] }; + this.patches_table = await this.synctable(query, [], this.patch_interval); this.assert_not_closed("init_patch_list -- after making synctable"); const update_has_unsaved_changes = debounce( @@ -2089,7 +2105,11 @@ export class SyncDoc extends EventEmitter { } //console.log 'saving patch with time ', time.valueOf() - const x = this.patches_table.set(obj, "none"); + let x = this.patches_table.set(obj, "none"); + if (x == null) { + // TODO: just for NATS right now! + x = fromJS(obj); + } const y = this.process_patch(x, undefined, undefined, patch); if (y != null) { assertDefined(this.patch_list); @@ -2318,11 +2338,15 @@ export class SyncDoc extends EventEmitter { // m below is an immutable map with keys the string that // is the JSON version of the primary key // [string_id, timestamp, user_number]. - const m: Map | undefined = this.patches_table.get(); + let m: Map | undefined = this.patches_table.get(); if (m == null) { // won't happen because of assert above. throw Error("patches_table must be initialized"); } + if (!Map.isMap(m)) { + // TODO: this is just for proof of concept NATS!! + m = fromJS(m); + } const v: Patch[] = []; m.forEach((x, _) => { const p = this.process_patch(x, time0, time1); @@ -2407,7 +2431,7 @@ export class SyncDoc extends EventEmitter { } } } - } + }; public get_last_save_to_disk_time = (): Date => { return this.last_save_to_disk_time; @@ -2874,6 +2898,10 @@ export class SyncDoc extends EventEmitter { /* Initiates a save of file to disk, then waits for the state to change. */ public save_to_disk = async (): Promise => { + if (this.path == "nats.txt") { + // TODO: nats! + return; + } if (this.state != "ready") { // We just make save_to_disk a successful // no operation, if the document is either @@ -3176,8 +3204,12 @@ export class SyncDoc extends EventEmitter { dbg("queue size = ", this.patch_update_queue.length); const v: Patch[] = []; for (const key of this.patch_update_queue) { - const x = this.patches_table.get(key); + let x = this.patches_table.get(key); if (x != null) { + if (!Map.isMap(x)) { + // NATS TODO! + x = fromJS(x); + } // may be null, e.g., when deleted. const t = x.get("time"); // Only need to process patches that we didn't diff --git a/src/packages/sync/editor/generic/types.ts b/src/packages/sync/editor/generic/types.ts index 45907bc099..e083b22d42 100644 --- a/src/packages/sync/editor/generic/types.ts +++ b/src/packages/sync/editor/generic/types.ts @@ -95,6 +95,8 @@ export interface ProjectClient extends EventEmitter { id?: string, ) => Promise; + synctable_nats: (query: any, obj?) => Promise; + // account_id or project_id or compute_server_id (encoded as a UUID - use decodeUUIDtoNum to decode) client_id: () => string; diff --git a/src/packages/sync/editor/string/test/client-test.ts b/src/packages/sync/editor/string/test/client-test.ts index b3755fdeab..49e3d13ad4 100644 --- a/src/packages/sync/editor/string/test/client-test.ts +++ b/src/packages/sync/editor/string/test/client-test.ts @@ -171,6 +171,10 @@ export class Client extends EventEmitter implements Client0 { throw Error("not implemented"); } + async synctable_nats(_query: any): Promise { + throw Error("not implemented"); + } + // account_id or project_id public client_id(): string { return this._client_id; diff --git a/src/packages/util/nats/synctable-stream.ts b/src/packages/util/nats/synctable-stream.ts index 2c62addb40..099ab4e0a4 100644 --- a/src/packages/util/nats/synctable-stream.ts +++ b/src/packages/util/nats/synctable-stream.ts @@ -13,8 +13,11 @@ import { jetstreamManager, jetstream } from "@nats-io/jetstream"; import sha1 from "sha1"; import jsonStableStringify from "json-stable-stringify"; import { keys } from "lodash"; -import { isValidUUID } from "@cocalc/util/misc"; +import { cmp_Date, is_array, isValidUUID } from "@cocalc/util/misc"; import { client_db } from "@cocalc/util/db-schema/client-db"; +import { EventEmitter } from "events"; + +export type State = "disconnected" | "connected" | "closed"; interface NatsEnv { nc; // nats connection @@ -33,27 +36,31 @@ function toKey(x): string | undefined { } } -export class SyncTableStream { - private kv?; +export class SyncTableStream extends EventEmitter { private nc; private jc; private sha1; private table; private primaryKeys: string[]; - private primaryKeysSet: Set; - private fields: string[]; private project_id: string; private streamName: string; private path: string; private subject: string; + private string_id: string; + private data: any = {}; private consumer?; + private state: State = "disconnected"; constructor({ query, env }: { query; env: NatsEnv }) { + super(); this.sha1 = env.sha1 ?? sha1; this.nc = env.nc; this.jc = env.jc; const table = keys(query)[0]; this.table = table; + if (table != "patches") { + throw Error("only the patches table is supported"); + } this.project_id = query[table][0].project_id; if (!isValidUUID(this.project_id)) { throw Error("query MUST specify a valid project_id"); @@ -62,14 +69,12 @@ export class SyncTableStream { if (!this.path) { throw Error("path MUST be specified"); } - query[table][0].string_id = this.sha1(`${this.project_id}${this.path}`); + query[table][0].string_id = this.string_id = this.sha1( + `${this.project_id}${this.path}`, + ); this.streamName = `${this.table}-${query[table][0].string_id}`; this.subject = `project.${this.project_id}.${this.table}.${query[table][0].string_id}`; this.primaryKeys = client_db.primary_keys(table); - this.primaryKeysSet = new Set(this.primaryKeys); - this.fields = keys(query[table][0]).filter( - (field) => !this.primaryKeysSet.has(field), - ); } private createStream = async () => { @@ -102,37 +107,59 @@ export class SyncTableStream { init = async () => { await this.createStream(); this.consumer = await this.getConsumer(); + await this.readData(); + this.set_state("connected"); this.getMessages(); }; - private primaryString = (obj): string => {}; + private set_state = (state: State): void => { + this.state = state; + this.emit(state); + }; - private natObjectKey = (obj): string => {}; + get_state = () => { + return this.state; + }; - private getKey = (obj, field?: string): string => {}; + private primaryString = (obj): string => { + const obj2 = { ...obj, string_id: this.string_id }; + return toKey(this.primaryKeys.map((pk) => obj2[pk]))!; + }; private publish = (mesg) => { + console.log("publishing ", { subject: this.subject, mesg }); this.nc.publish(this.subject, this.jc.encode(mesg)); }; - set = async (obj) => { - delete obj["string_id"]; // redundant - this.publish(obj); + set = (obj) => { + console.log("set", obj); + // delete string_id since it is redundant info + const { string_id, ...obj2 } = obj; + this.publish(obj2); }; - private handle = (mesg) => { - const x = this.jc.decode(mesg.data); - console.log(x); + private handle = (mesg, changeEvent: boolean) => { + if (this.state == "closed") { + return true; + } + const obj = this.jc.decode(mesg.data); + const key = this.primaryString(obj); + this.data[key] = { ...obj, time: new Date(obj.time) }; + if (changeEvent) { + this.emit("change", [key]); + } return false; }; - private getMessages = async () => { + // load initial data + private readData = async () => { const consumer = this.consumer!; const messages = await consumer.fetch({ max_messages: 100000, + expires: 1000, }); for await (const mesg of messages) { - if (this.handle(mesg)) { + if (this.handle(mesg, false)) { return; } if (mesg.info.pending == 0) { @@ -140,14 +167,60 @@ export class SyncTableStream { break; } } + }; + + // listen for new data + private getMessages = async () => { + const consumer = this.consumer!; for await (const mesg of await consumer.consume()) { - if (this.handle(mesg)) { + if (this.handle(mesg, true)) { return; } } }; - get = async (obj?, field?) => { - console.log("get: TODO"); + get = (obj?) => { + if (obj == null) { + // CAREFUL + return this.data; + } + if (typeof obj == "string") { + return this.data[obj]; + } + if (is_array(obj)) { + const x: any = {}; + for (const key of obj) { + x[this.primaryString(key)] = this.get(key); + } + return x; + } + let key; + if (typeof obj == "object") { + key = this.primaryString(obj); + } else { + key = `${key}`; + } + return this.data[key]; + }; + + getSortedTimes = () => { + return Object.values(this.data) + .map(({ time }) => time) + .sort(cmp_Date); + }; + + close = () => { + if (this.state === "closed") { + // already closed + return; + } + this.set_state("closed"); + }; + + // no-op because we always immediately publish changes on set. + save = () => {}; + has_uncommitted_changes = () => { + // todo - if disconnected (?) + return false; }; } diff --git a/src/packages/util/nats/synctable.ts b/src/packages/util/nats/synctable.ts new file mode 100644 index 0000000000..fff6edc9ec --- /dev/null +++ b/src/packages/util/nats/synctable.ts @@ -0,0 +1,13 @@ +import { SyncTableKV } from "./synctable-kv"; +import { SyncTableStream } from "./synctable-stream"; +import { keys } from "lodash"; + +export type SyncTable = SyncTableKV | SyncTableStream; + +export function createSyncTable({ query, env }) { + const table = keys(query)[0]; + if (table == "patches") { + return new SyncTableStream({ query, env }); + } + return new SyncTableKV({ query, env }); +} From ad7807ce19f5425f933e73e72c299ce71418d2fb Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 25 Jan 2025 14:24:45 +0000 Subject: [PATCH 034/281] nats collab editing proof of concept -- fix some issues so it works --- src/packages/sync/editor/generic/sync-doc.ts | 17 ++++++++++++----- src/packages/util/nats/synctable-stream.ts | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index b6c9f63c96..f59e748ced 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -247,6 +247,8 @@ export class SyncDoc extends EventEmitter { // static because we want exactly one across all docs! private static computeServerManagerDoc?: SyncDoc; + private useNats: boolean; + constructor(opts: SyncOpts) { super(); if (opts.string_id === undefined) { @@ -273,6 +275,7 @@ export class SyncDoc extends EventEmitter { this[field] = opts[field]; } } + this.useNats = this.path.startsWith("nats/"); if (this.ephemeral) { // So the doctype written to the database reflects the // ephemeral state. Here ephemeral determines whether @@ -671,8 +674,12 @@ export class SyncDoc extends EventEmitter { // This gets used by clients that are connected to a backend // with state in the project (e.g., jupyter). Basically this // is a special websocket channel just for this syncdoc, which - // uses the cursors table. + // uses the patches table. public sendMessageToProject = async (data) => { + if (this.useNats) { + console.log("sendMessageToProject - IGNORING", data); + return; + } const send = this.patches_table?.sendMessageToProject; if (send == null || this.patches_table.channel == null) { throw Error("sending messages to project not available"); @@ -1242,7 +1249,7 @@ export class SyncDoc extends EventEmitter { options.push({ ephemeral: true }); } let synctable; - if (this.path == "nats.txt" && query.patches) { + if (this.useNats && query.patches) { synctable = await this.client.synctable_nats(query, { project_id: this.project_id, path: this.path, @@ -1333,7 +1340,7 @@ export class SyncDoc extends EventEmitter { // Used for internal debug logging private dbg = (f: string = ""): Function => { - if (this.path == "nats.txt") { + if (this.useNats) { return (...args) => console.log(f, ...args); } return this.client?.dbg(`SyncDoc('${this.path}').${f}`); @@ -1419,7 +1426,7 @@ export class SyncDoc extends EventEmitter { // wait until the syncstring table is ready to be // used (so extracted from archive, etc.), private async wait_until_fully_ready(): Promise { - if (this.path == "nats.txt") { + if (this.useNats) { return; } this.assert_not_closed("wait_until_fully_ready"); @@ -2898,7 +2905,7 @@ export class SyncDoc extends EventEmitter { /* Initiates a save of file to disk, then waits for the state to change. */ public save_to_disk = async (): Promise => { - if (this.path == "nats.txt") { + if (this.useNats) { // TODO: nats! return; } diff --git a/src/packages/util/nats/synctable-stream.ts b/src/packages/util/nats/synctable-stream.ts index 099ab4e0a4..7cd71c6420 100644 --- a/src/packages/util/nats/synctable-stream.ts +++ b/src/packages/util/nats/synctable-stream.ts @@ -127,14 +127,22 @@ export class SyncTableStream extends EventEmitter { }; private publish = (mesg) => { - console.log("publishing ", { subject: this.subject, mesg }); + // console.log("publishing ", { subject: this.subject, mesg }); this.nc.publish(this.subject, this.jc.encode(mesg)); }; set = (obj) => { - console.log("set", obj); + // console.log("set", obj); // delete string_id since it is redundant info + const key = this.primaryString(obj); + if (this.data[key] != null) { + // no changes to existing keys -- just ignore. + // TODO? + // console.log("set - skip", obj); + return; + } const { string_id, ...obj2 } = obj; + // console.log("set - publish", obj); this.publish(obj2); }; @@ -145,6 +153,9 @@ export class SyncTableStream extends EventEmitter { const obj = this.jc.decode(mesg.data); const key = this.primaryString(obj); this.data[key] = { ...obj, time: new Date(obj.time) }; + if (this.data[key].prev != null) { + this.data[key].prev = new Date(this.data[key].prev); + } if (changeEvent) { this.emit("change", [key]); } From f03c3b07e2a6f338e84bc359bb92dcea2f083ce3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 25 Jan 2025 14:36:40 +0000 Subject: [PATCH 035/281] nats sync: a little bit of saving --- src/packages/project/client.ts | 2 +- src/packages/sync/editor/generic/sync-doc.ts | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index 8364998649..946c9a0e67 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -27,7 +27,7 @@ import { join } from "node:path"; import { FileSystemClient } from "@cocalc/sync-client/lib/client-fs"; import { execute_code, uuidsha1 } from "@cocalc/backend/misc_node"; import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; -import { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; +import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; import type { ProjectClient as ProjectClientInterface } from "@cocalc/sync/editor/generic/types"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import * as synctable2 from "@cocalc/sync/table"; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index f59e748ced..b0a6648f19 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -2905,10 +2905,6 @@ export class SyncDoc extends EventEmitter { /* Initiates a save of file to disk, then waits for the state to change. */ public save_to_disk = async (): Promise => { - if (this.useNats) { - // TODO: nats! - return; - } if (this.state != "ready") { // We just make save_to_disk a successful // no operation, if the document is either From 2fa9fd20c7e6cc93db88a6e3892e1d63432d324e Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 25 Jan 2025 16:44:50 +0000 Subject: [PATCH 036/281] nats: one subject for all patches in a project --- docs/nats/devlog.md | 6 +++++- .../terminal-editor/connected-terminal.ts | 2 +- src/packages/util/nats/synctable-stream.ts | 11 +++++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index bea9d5dd1e..dfd7823f75 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -237,7 +237,11 @@ Plan. - [x] Use a kv store hosted on nats to trac syncstring objects as before. This means anybody can participate \(browser, compute server, project\) without any need to contact the database, hence eliminating all proxying! -#now Next Goal \- collaborative file editing! This requires implementing the "ordered patches list" but on jetstream. Similar to the nats SyncTable I wrote yesterday, except will use jetstream directly, since it is an event stream, after all. +[x] Next Goal \- collaborative file editing \-\- some sort of "proof of concept"! This requires implementing the "ordered patches list" but on jetstream. Similar to the nats SyncTable I wrote yesterday, except will use jetstream directly, since it is an event stream, after all. + +- [ ] synctable\-stream: change to one big stream for the whole project but **consume** a specific subject in that stream? + +[ ] cursors \- an ephemeral table --- diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 644c667dd9..11a3030152 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -299,7 +299,7 @@ export class Terminal { }; async connect(): Promise { - if (this.path == "nats.term") { + if (this.path.startsWith("nats/")) { return await this.connectNats(); } this.assert_not_closed(); diff --git a/src/packages/util/nats/synctable-stream.ts b/src/packages/util/nats/synctable-stream.ts index 7cd71c6420..242e0da978 100644 --- a/src/packages/util/nats/synctable-stream.ts +++ b/src/packages/util/nats/synctable-stream.ts @@ -44,6 +44,7 @@ export class SyncTableStream extends EventEmitter { private primaryKeys: string[]; private project_id: string; private streamName: string; + private streamSubject: string; private path: string; private subject: string; private string_id: string; @@ -72,7 +73,8 @@ export class SyncTableStream extends EventEmitter { query[table][0].string_id = this.string_id = this.sha1( `${this.project_id}${this.path}`, ); - this.streamName = `${this.table}-${query[table][0].string_id}`; + this.streamName = `project-${this.project_id}-${this.table}`; + this.streamSubject = `project.${this.project_id}.${this.table}.>`; this.subject = `project.${this.project_id}.${this.table}.${query[table][0].string_id}`; this.primaryKeys = client_db.primary_keys(table); } @@ -82,13 +84,14 @@ export class SyncTableStream extends EventEmitter { try { await jsm.streams.add({ name: this.streamName, - subjects: [this.subject], + subjects: [this.streamSubject], compression: "s2", }); - } catch (_err) { + } catch (err) { + console.log("createStream", err); // probably already exists await jsm.streams.update(this.streamName, { - subjects: [this.subject], + subjects: [this.streamSubject], compression: "s2" as any, }); } From fb4b4a4fa2635fe43fa43fcc07670f0c248ae594 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 25 Jan 2025 17:40:52 +0000 Subject: [PATCH 037/281] nats: create a new package --- docs/nats/devlog.md | 2 +- src/packages/frontend/client/nats.ts | 2 +- src/packages/frontend/package.json | 3 +- src/packages/frontend/tsconfig.json | 1 + src/packages/nats/package.json | 39 ++++ .../{util/nats => nats/sync}/synctable-kv.ts | 0 .../nats => nats/sync}/synctable-stream.ts | 0 .../{util/nats => nats/sync}/synctable.ts | 0 src/packages/nats/tsconfig.json | 8 + src/packages/next/tsconfig.json | 6 +- src/packages/pnpm-lock.yaml | 197 ++++++++++-------- src/packages/project/nats/synctable.ts | 2 +- src/packages/project/package.json | 1 + src/packages/project/tsconfig.json | 1 + src/packages/util/package.json | 5 +- src/workspaces.py | 1 + 16 files changed, 173 insertions(+), 95 deletions(-) create mode 100644 src/packages/nats/package.json rename src/packages/{util/nats => nats/sync}/synctable-kv.ts (100%) rename src/packages/{util/nats => nats/sync}/synctable-stream.ts (100%) rename src/packages/{util/nats => nats/sync}/synctable.ts (100%) create mode 100644 src/packages/nats/tsconfig.json diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index dfd7823f75..cf77b5e794 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -239,7 +239,7 @@ Plan. [x] Next Goal \- collaborative file editing \-\- some sort of "proof of concept"! This requires implementing the "ordered patches list" but on jetstream. Similar to the nats SyncTable I wrote yesterday, except will use jetstream directly, since it is an event stream, after all. -- [ ] synctable\-stream: change to one big stream for the whole project but **consume** a specific subject in that stream? +- [x] synctable\-stream: change to one big stream for the whole project but **consume** a specific subject in that stream? [ ] cursors \- an ephemeral table diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 16726cc203..e07369f0fb 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -5,7 +5,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { join } from "path"; import { redux } from "../app-framework"; import * as jetstream from "@nats-io/jetstream"; -import { createSyncTable, type SyncTable } from "@cocalc/util/nats/synctable"; +import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; import { parse_query } from "@cocalc/sync/table/util"; import sha1 from "sha1"; import { keys } from "lodash"; diff --git a/src/packages/frontend/package.json b/src/packages/frontend/package.json index 1996a66bb0..8ea6303873 100644 --- a/src/packages/frontend/package.json +++ b/src/packages/frontend/package.json @@ -43,9 +43,10 @@ "@cocalc/comm": "workspace:*", "@cocalc/frontend": "workspace:*", "@cocalc/jupyter": "workspace:*", - "@cocalc/local-storage-lru": "^2.4.3", + "@cocalc/nats": "workspace:*", "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", + "@cocalc/local-storage-lru": "^2.4.3", "@cocalc/widgets": "^1.2.0", "@dnd-kit/core": "^6.0.7", "@dnd-kit/modifiers": "^6.0.1", diff --git a/src/packages/frontend/tsconfig.json b/src/packages/frontend/tsconfig.json index 1796304616..c91da6e99e 100644 --- a/src/packages/frontend/tsconfig.json +++ b/src/packages/frontend/tsconfig.json @@ -28,6 +28,7 @@ "references": [ { "path": "../comm" }, { "path": "../jupyter" }, + { "path": "../nats" }, { "path": "../sync" }, { "path": "../util" } ] diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json new file mode 100644 index 0000000000..0317f7cf66 --- /dev/null +++ b/src/packages/nats/package.json @@ -0,0 +1,39 @@ +{ + "name": "@cocalc/nats", + "version": "1.0.0", + "description": "CoCalc NATS integration code. Usable by both nodejs and browser.", + "exports": { + "./sync/*": "./dist/sync/*.js" + }, + "scripts": { + "preinstall": "npx only-allow pnpm", + "build": "pnpm exec tsc --build", + "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", + "test": "pnpm exec jest", + "prepublishOnly": "pnpm test" + }, + "files": ["dist/**", "README.md", "package.json"], + "author": "SageMath, Inc.", + "keywords": ["utilities", "nats", "cocalc"], + "license": "SEE LICENSE.md", + "dependencies": { + "@cocalc/nats": "workspace:*", + "@cocalc/util": "workspace:*", + "@nats-io/jetstream": "3.0.0-36", + "@nats-io/kv": "3.0.0-30", + "events": "3.3.0", + "json-stable-stringify": "^1.0.1", + "lodash": "^4.17.21", + "sha1": "^1.1.1" + }, + "devDependencies": { + "@types/json-stable-stringify": "^1.0.32", + "@types/lodash": "^4.14.202", + "@types/node": "^18.16.14" + }, + "repository": { + "type": "git", + "url": "https://github.com/sagemathinc/cocalc" + }, + "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/nats" +} diff --git a/src/packages/util/nats/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts similarity index 100% rename from src/packages/util/nats/synctable-kv.ts rename to src/packages/nats/sync/synctable-kv.ts diff --git a/src/packages/util/nats/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts similarity index 100% rename from src/packages/util/nats/synctable-stream.ts rename to src/packages/nats/sync/synctable-stream.ts diff --git a/src/packages/util/nats/synctable.ts b/src/packages/nats/sync/synctable.ts similarity index 100% rename from src/packages/util/nats/synctable.ts rename to src/packages/nats/sync/synctable.ts diff --git a/src/packages/nats/tsconfig.json b/src/packages/nats/tsconfig.json new file mode 100644 index 0000000000..98172f1f9b --- /dev/null +++ b/src/packages/nats/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "rootDir": "./", + "outDir": "dist" + }, + "exclude": ["node_modules", "dist", "test"] +} diff --git a/src/packages/next/tsconfig.json b/src/packages/next/tsconfig.json index 3de6828c6a..d863adeded 100644 --- a/src/packages/next/tsconfig.json +++ b/src/packages/next/tsconfig.json @@ -30,13 +30,15 @@ "**/*.tsx", "software-inventory/*.json", "components/landing/*.json", - "locales/*/*.json" -, "locales/en/common.ts" ], + "locales/*/*.json", + "locales/en/common.ts" + ], "exclude": ["node_modules", "public", "styles", "dist"], "references": [ { "path": "../backend" }, { "path": "../database" }, { "path": "../frontend" }, + { "path": "../nats" }, { "path": "../server" }, { "path": "../util" } ] diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index a7951b2bb5..03e7d436d6 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -31,10 +31,10 @@ importers: version: 5.0.0 jest: specifier: ^29.7.0 - version: 29.7.0(@types/node@18.19.55) + version: 29.7.0(@types/node@18.19.71) ts-jest: specifier: ^29.2.3 - version: 29.2.5(@babel/core@7.25.8)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.8))(jest@29.7.0(@types/node@18.19.55))(typescript@5.7.3) + version: 29.2.5(@babel/core@7.25.8)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.8))(jest@29.7.0(@types/node@18.19.71))(typescript@5.7.3) typescript: specifier: ^5.7.3 version: 5.7.3 @@ -262,6 +262,9 @@ importers: '@cocalc/local-storage-lru': specifier: ^2.4.3 version: 2.5.0 + '@cocalc/nats': + specifier: workspace:* + version: link:../nats '@cocalc/sync': specifier: workspace:* version: link:../sync @@ -1040,6 +1043,43 @@ importers: specifier: ^18.16.14 version: 18.19.50 + nats: + dependencies: + '@cocalc/nats': + specifier: workspace:* + version: 'link:' + '@cocalc/util': + specifier: workspace:* + version: link:../util + '@nats-io/jetstream': + specifier: 3.0.0-36 + version: 3.0.0-36 + '@nats-io/kv': + specifier: 3.0.0-30 + version: 3.0.0-30 + events: + specifier: 3.3.0 + version: 3.3.0 + json-stable-stringify: + specifier: ^1.0.1 + version: 1.1.1 + lodash: + specifier: ^4.17.21 + version: 4.17.21 + sha1: + specifier: ^1.1.1 + version: 1.1.1 + devDependencies: + '@types/json-stable-stringify': + specifier: ^1.0.32 + version: 1.0.36 + '@types/lodash': + specifier: ^4.14.202 + version: 4.17.9 + '@types/node': + specifier: ^18.16.14 + version: 18.19.71 + next: dependencies: '@ant-design/icons': @@ -1205,6 +1245,9 @@ importers: '@cocalc/jupyter': specifier: workspace:* version: link:../jupyter + '@cocalc/nats': + specifier: workspace:* + version: link:../nats '@cocalc/primus-multiplex': specifier: ^1.1.0 version: 1.1.0 @@ -2013,12 +2056,6 @@ importers: '@isaacs/ttlcache': specifier: ^1.2.1 version: 1.4.1 - '@nats-io/jetstream': - specifier: 3.0.0-36 - version: 3.0.0-36 - '@nats-io/kv': - specifier: 3.0.0-30 - version: 3.0.0-30 '@types/debug': specifier: ^4.1.12 version: 4.1.12 @@ -2033,7 +2070,7 @@ importers: version: 1.11.13 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0 decimal.js-light: specifier: ^2.5.1 version: 2.5.1 @@ -4740,12 +4777,6 @@ packages: '@types/node@18.19.50': resolution: {integrity: sha512-xonK+NRrMBRtkL1hVCc3G+uXtjh1Al4opBLjqVmipe5ZAaBYWW6cNAiBVZ1BvmkBhep698rP3UM3aRAdSALuhg==} - '@types/node@18.19.55': - resolution: {integrity: sha512-zzw5Vw52205Zr/nmErSEkN5FLqXPuKX/k5d1D7RKHATGqU7y6YfX9QxZraUzUrFGqH6XzOzG196BC35ltJC4Cw==} - - '@types/node@18.19.64': - resolution: {integrity: sha512-955mDqvO2vFf/oL7V3WiUtiz+BugyX8uVbaT2H8oj3+8dRyH2FLiNdowe7eNqRM7IOIZvzDH76EoAT+gwm6aIQ==} - '@types/node@18.19.71': resolution: {integrity: sha512-evXpcgtZm8FY4jqBSN8+DmOTcVkkvTmAayeo4Wf3m1xAruyVGzGuDh/Fb/WWX2yLItUiho42ozyJjB0dw//Tkw==} @@ -12931,7 +12962,7 @@ snapshots: '@anthropic-ai/sdk@0.32.1(encoding@0.1.13)': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -12983,7 +13014,7 @@ snapshots: '@babel/traverse': 7.25.6 '@babel/types': 7.25.6 convert-source-map: 2.0.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -13526,7 +13557,7 @@ snapshots: '@babel/parser': 7.25.6 '@babel/template': 7.25.0 '@babel/types': 7.25.6 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0 globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -14235,7 +14266,7 @@ snapshots: '@jest/console@29.7.0': dependencies: '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.71 chalk: 4.1.2 jest-message-util: 29.7.0 jest-util: 29.7.0 @@ -14248,14 +14279,14 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.55 + '@types/node': 18.19.71 ansi-escapes: 4.3.2 chalk: 4.1.2 ci-info: 3.9.0 exit: 0.1.2 graceful-fs: 4.2.11 jest-changed-files: 29.7.0 - jest-config: 29.7.0(@types/node@18.19.55) + jest-config: 29.7.0(@types/node@18.19.71) jest-haste-map: 29.7.0 jest-message-util: 29.7.0 jest-regex-util: 29.6.3 @@ -14280,7 +14311,7 @@ snapshots: dependencies: '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.71 jest-mock: 29.7.0 '@jest/expect-utils@29.7.0': @@ -14298,7 +14329,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@sinonjs/fake-timers': 10.3.0 - '@types/node': 18.19.64 + '@types/node': 18.19.71 jest-message-util: 29.7.0 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -14320,7 +14351,7 @@ snapshots: '@jest/transform': 29.7.0 '@jest/types': 29.6.3 '@jridgewell/trace-mapping': 0.3.25 - '@types/node': 18.19.64 + '@types/node': 18.19.71 chalk: 4.1.2 collect-v8-coverage: 1.0.2 exit: 0.1.2 @@ -14389,7 +14420,7 @@ snapshots: dependencies: '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.1 - '@types/node': 18.19.50 + '@types/node': 18.19.71 '@types/yargs': 15.0.19 chalk: 4.1.2 @@ -14398,7 +14429,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.1 - '@types/node': 18.19.55 + '@types/node': 18.19.71 '@types/yargs': 17.0.24 chalk: 4.1.2 @@ -15637,11 +15668,11 @@ snapshots: '@types/body-parser@1.19.5': dependencies: '@types/connect': 3.4.35 - '@types/node': 18.19.50 + '@types/node': 18.19.71 '@types/bonjour@3.5.13': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/caseless@0.12.5': {} @@ -15654,11 +15685,11 @@ snapshots: '@types/connect-history-api-fallback@1.5.4': dependencies: '@types/express-serve-static-core': 5.0.1 - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/connect@3.4.35': dependencies: - '@types/node': 18.19.55 + '@types/node': 18.19.71 '@types/cookie@0.3.3': {} @@ -15691,14 +15722,14 @@ snapshots: '@types/express-serve-static-core@4.19.0': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/qs': 6.9.17 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 '@types/express-serve-static-core@5.0.1': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/qs': 6.9.17 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 @@ -15736,11 +15767,11 @@ snapshots: '@types/glob@7.2.0': dependencies: '@types/minimatch': 5.1.2 - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/graceful-fs@4.1.9': dependencies: - '@types/node': 18.19.55 + '@types/node': 18.19.71 '@types/hast@2.3.10': dependencies: @@ -15790,11 +15821,11 @@ snapshots: '@types/keyv@3.1.4': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/ldapjs@2.2.5': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/linkify-it@5.0.0': {} @@ -15804,7 +15835,7 @@ snapshots: '@types/lz4@0.6.4': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.71 '@types/mapbox__point-geometry@0.1.4': {} @@ -15841,7 +15872,7 @@ snapshots: '@types/node-fetch@2.6.11': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 form-data: 4.0.0 '@types/node-fetch@2.6.12': @@ -15851,11 +15882,11 @@ snapshots: '@types/node-forge@1.3.11': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/node-zendesk@2.0.15': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.71 '@types/node@10.14.22': {} @@ -15863,14 +15894,6 @@ snapshots: dependencies: undici-types: 5.26.5 - '@types/node@18.19.55': - dependencies: - undici-types: 5.26.5 - - '@types/node@18.19.64': - dependencies: - undici-types: 5.26.5 - '@types/node@18.19.71': dependencies: undici-types: 5.26.5 @@ -15879,13 +15902,13 @@ snapshots: '@types/nodemailer@6.4.16': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.71 '@types/normalize-package-data@2.4.1': {} '@types/oauth@0.9.1': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/parse5@6.0.3': {} @@ -15953,13 +15976,13 @@ snapshots: '@types/request@2.48.12': dependencies: '@types/caseless': 0.12.5 - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/tough-cookie': 4.0.5 form-data: 2.5.1 '@types/responselike@1.0.3': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/retry@0.12.0': {} @@ -15976,7 +15999,7 @@ snapshots: '@types/send@0.17.4': dependencies: '@types/mime': 1.3.5 - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/serve-index@1.9.4': dependencies: @@ -15985,19 +16008,19 @@ snapshots: '@types/serve-static@1.15.0': dependencies: '@types/mime': 3.0.1 - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/serve-static@1.15.7': dependencies: '@types/http-errors': 2.0.4 - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/send': 0.17.4 '@types/sizzle@2.3.3': {} '@types/sockjs@0.3.36': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/stack-utils@2.0.3': {} @@ -16028,20 +16051,20 @@ snapshots: '@types/ws@8.5.13': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/xml-crypto@1.4.6': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 xpath: 0.0.27 '@types/xml-encryption@1.2.4': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/xml2js@0.4.14': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/yargs-parser@21.0.0': {} @@ -17549,13 +17572,13 @@ snapshots: - supports-color - ts-node - create-jest@29.7.0(@types/node@18.19.55): + create-jest@29.7.0(@types/node@18.19.71): dependencies: '@jest/types': 29.6.3 chalk: 4.1.2 exit: 0.1.2 graceful-fs: 4.2.11 - jest-config: 29.7.0(@types/node@18.19.55) + jest-config: 29.7.0(@types/node@18.19.71) jest-util: 29.7.0 prompts: 2.4.2 transitivePeerDependencies: @@ -18036,6 +18059,10 @@ snapshots: dependencies: ms: 2.1.2 + debug@4.4.0: + dependencies: + ms: 2.1.3 + debug@4.4.0(supports-color@8.1.1): dependencies: ms: 2.1.3 @@ -20360,7 +20387,7 @@ snapshots: istanbul-lib-source-maps@4.0.1: dependencies: - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0 istanbul-lib-coverage: 3.2.2 source-map: 0.6.1 transitivePeerDependencies: @@ -20416,7 +20443,7 @@ snapshots: '@jest/expect': 29.7.0 '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.71 chalk: 4.1.2 co: 4.6.0 dedent: 1.5.3 @@ -20455,16 +20482,16 @@ snapshots: - supports-color - ts-node - jest-cli@29.7.0(@types/node@18.19.55): + jest-cli@29.7.0(@types/node@18.19.71): dependencies: '@jest/core': 29.7.0 '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 chalk: 4.1.2 - create-jest: 29.7.0(@types/node@18.19.55) + create-jest: 29.7.0(@types/node@18.19.71) exit: 0.1.2 import-local: 3.2.0 - jest-config: 29.7.0(@types/node@18.19.55) + jest-config: 29.7.0(@types/node@18.19.71) jest-util: 29.7.0 jest-validate: 29.7.0 yargs: 17.7.2 @@ -20504,7 +20531,7 @@ snapshots: - babel-plugin-macros - supports-color - jest-config@29.7.0(@types/node@18.19.55): + jest-config@29.7.0(@types/node@18.19.71): dependencies: '@babel/core': 7.25.8 '@jest/test-sequencer': 29.7.0 @@ -20529,7 +20556,7 @@ snapshots: slash: 3.0.0 strip-json-comments: 3.1.1 optionalDependencies: - '@types/node': 18.19.55 + '@types/node': 18.19.71 transitivePeerDependencies: - babel-plugin-macros - supports-color @@ -20565,7 +20592,7 @@ snapshots: '@jest/environment': 29.7.0 '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.71 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -20577,7 +20604,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@types/graceful-fs': 4.1.9 - '@types/node': 18.19.64 + '@types/node': 18.19.71 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11 @@ -20635,7 +20662,7 @@ snapshots: jest-mock@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.71 jest-util: 29.7.0 jest-pnp-resolver@1.2.3(jest-resolve@29.7.0): @@ -20672,7 +20699,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.71 chalk: 4.1.2 emittery: 0.13.1 graceful-fs: 4.2.11 @@ -20700,7 +20727,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.71 chalk: 4.1.2 cjs-module-lexer: 1.2.3 collect-v8-coverage: 1.0.2 @@ -20746,7 +20773,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 18.19.55 + '@types/node': 18.19.71 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -20765,7 +20792,7 @@ snapshots: dependencies: '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.71 ansi-escapes: 4.3.2 chalk: 4.1.2 emittery: 0.13.1 @@ -20780,7 +20807,7 @@ snapshots: jest-worker@29.7.0: dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 jest-util: 29.7.0 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -20797,12 +20824,12 @@ snapshots: - supports-color - ts-node - jest@29.7.0(@types/node@18.19.55): + jest@29.7.0(@types/node@18.19.71): dependencies: '@jest/core': 29.7.0 '@jest/types': 29.6.3 import-local: 3.2.0 - jest-cli: 29.7.0(@types/node@18.19.55) + jest-cli: 29.7.0(@types/node@18.19.71) transitivePeerDependencies: - '@types/node' - babel-plugin-macros @@ -21832,7 +21859,7 @@ snapshots: node-mocks-http@1.16.0: dependencies: '@types/express': 4.17.21 - '@types/node': 18.19.50 + '@types/node': 18.19.71 accepts: 1.3.8 content-disposition: 0.5.4 depd: 1.1.2 @@ -22067,7 +22094,7 @@ snapshots: openai@4.78.1(encoding@0.1.13)(zod@3.23.8): dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.71 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -22817,7 +22844,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 18.19.64 + '@types/node': 18.19.71 long: 5.2.3 protocol-buffers-schema@3.6.0: {} @@ -24414,7 +24441,7 @@ snapshots: stripe@17.5.0: dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.71 qs: 6.14.0 strnum@1.0.5: {} @@ -24753,12 +24780,12 @@ snapshots: ts-dedent@2.2.0: {} - ts-jest@29.2.5(@babel/core@7.25.8)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.8))(jest@29.7.0(@types/node@18.19.55))(typescript@5.7.3): + ts-jest@29.2.5(@babel/core@7.25.8)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.8))(jest@29.7.0(@types/node@18.19.71))(typescript@5.7.3): dependencies: bs-logger: 0.2.6 ejs: 3.1.10 fast-json-stable-stringify: 2.1.0 - jest: 29.7.0(@types/node@18.19.55) + jest: 29.7.0(@types/node@18.19.71) jest-util: 29.7.0 json5: 2.2.3 lodash.memoize: 4.1.2 diff --git a/src/packages/project/nats/synctable.ts b/src/packages/project/nats/synctable.ts index e27d6963b5..6fd701fd41 100644 --- a/src/packages/project/nats/synctable.ts +++ b/src/packages/project/nats/synctable.ts @@ -2,7 +2,7 @@ import getConnection from "./connection"; import { project_id } from "@cocalc/project/data"; import { JSONCodec } from "nats"; import { sha1 } from "@cocalc/backend/sha1"; -import { createSyncTable, type SyncTable } from "@cocalc/util/nats/synctable"; +import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; import { parse_query } from "@cocalc/sync/table/util"; import { keys } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; diff --git a/src/packages/project/package.json b/src/packages/project/package.json index 6f9ac308f7..aa7d47a5c8 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -23,6 +23,7 @@ "@cocalc/comm": "workspace:*", "@cocalc/frontend": "workspace:*", "@cocalc/jupyter": "workspace:*", + "@cocalc/nats": "workspace:*", "@cocalc/primus-multiplex": "^1.1.0", "@cocalc/primus-responder": "^1.0.5", "@cocalc/project": "workspace:*", diff --git a/src/packages/project/tsconfig.json b/src/packages/project/tsconfig.json index 4e6d052ed7..39bcee4920 100644 --- a/src/packages/project/tsconfig.json +++ b/src/packages/project/tsconfig.json @@ -13,6 +13,7 @@ { "path": "../comm" }, { "path": "../frontend" }, { "path": "../jupyter" }, + { "path": "../nats" }, { "path": "../sync" }, { "path": "../sync-client" }, { "path": "../sync-fs" }, diff --git a/src/packages/util/package.json b/src/packages/util/package.json index 5fbee209c9..aadc9db6f4 100644 --- a/src/packages/util/package.json +++ b/src/packages/util/package.json @@ -12,8 +12,7 @@ "./sync/table": "./dist/sync/table/index.js", "./sync/editor/db": "./dist/sync/editor/db/index.js", "./licenses/purchase/*": "./dist/licenses/purchase/*.js", - "./redux/*": "./dist/redux/*.js", - "./nats/*": "./dist/nats/*.js" + "./redux/*": "./dist/redux/*.js" }, "scripts": { "preinstall": "npx only-allow pnpm", @@ -41,8 +40,6 @@ "@ant-design/colors": "^6.0.0", "@cocalc/util": "workspace:*", "@isaacs/ttlcache": "^1.2.1", - "@nats-io/jetstream": "3.0.0-36", - "@nats-io/kv": "3.0.0-30", "@types/debug": "^4.1.12", "async": "^1.5.2", "awaiting": "^3.0.0", diff --git a/src/workspaces.py b/src/workspaces.py index bd4201d6a0..1079a38d11 100755 --- a/src/workspaces.py +++ b/src/workspaces.py @@ -109,6 +109,7 @@ def all_packages() -> List[str]: 'packages/sync', 'packages/sync-client', 'packages/sync-fs', + 'packages/nats', 'packages/backend', 'packages/api-client', 'packages/jupyter', From 419067f94bf1a1b42c3e629bf78cdbf1cbd1f0e3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 25 Jan 2025 19:13:24 +0000 Subject: [PATCH 038/281] nats: work in progress figuring out a better type-checked way to do RPC calls --- src/packages/database/settings/customize.ts | 60 ++-------------- src/packages/frontend/client/nats.ts | 33 +++++---- src/packages/nats/api/index.ts | 18 +++++ src/packages/nats/package.json | 15 +++- src/packages/nats/tsconfig.json | 3 +- src/packages/pnpm-lock.yaml | 15 ++-- src/packages/server/nats/api.ts | 72 +++++++------------ src/packages/server/package.json | 1 + src/packages/server/tsconfig.json | 1 + .../util/db-schema/server-settings.ts | 53 ++++++++++++++ 10 files changed, 142 insertions(+), 129 deletions(-) create mode 100644 src/packages/nats/api/index.ts diff --git a/src/packages/database/settings/customize.ts b/src/packages/database/settings/customize.ts index f5b649c0f2..549431f49b 100644 --- a/src/packages/database/settings/customize.ts +++ b/src/packages/database/settings/customize.ts @@ -4,65 +4,13 @@ */ import getStrategies from "@cocalc/database/settings/get-sso-strategies"; -import { - KUCALC_COCALC_COM, - KucalcValues, -} from "@cocalc/util/db-schema/site-defaults"; -import { Strategy } from "@cocalc/util/types/sso"; +import { KUCALC_COCALC_COM } from "@cocalc/util/db-schema/site-defaults"; +import type { Strategy } from "@cocalc/util/types/sso"; import { ServerSettings, getServerSettings } from "./server-settings"; import siteURL from "./site-url"; import { copy_with } from "@cocalc/util/misc"; - -export interface Customize { - siteName?: string; - siteDescription?: string; - organizationName?: string; - organizationEmail?: string; - organizationURL?: string; - termsOfServiceURL?: string; - helpEmail?: string; - contactEmail?: string; - isCommercial?: boolean; - kucalc?: KucalcValues; - sshGateway?: boolean; - sshGatewayDNS?: string; - logoSquareURL?: string; - logoRectangularURL?: string; - splashImage?: string; - indexInfo?: string; - indexTagline?: string; - imprint?: string; - policies?: string; - shareServer?: boolean; - landingPages?: boolean; - dns?: string; - siteURL?: string; - googleAnalytics?: string; - anonymousSignup?: boolean; - anonymousSignupLicensedShares?: boolean; - emailSignup?: boolean; - accountCreationInstructions?: string; - zendesk?: boolean; // true if zendesk support is configured. - stripePublishableKey?: string; - imprint_html?: string; - policies_html?: string; - reCaptchaKey?: string; - sandboxProjectsEnabled?: boolean; - sandboxProjectId?: string; - verifyEmailAddresses?: boolean; - strategies?: Strategy[]; - openaiEnabled?: boolean; - googleVertexaiEnabled?: boolean; - mistralEnabled?: boolean; - anthropicEnabled?: boolean; - ollamaEnabled?: boolean; - neuralSearchEnabled?: boolean; - jupyterApiEnabled?: boolean; - computeServersEnabled?: boolean; - cloudFilesystemsEnabled?: boolean; - githubProjectId?: string; - support?: string; -} +import type { Customize } from "@cocalc/util/db-schema/server-settings"; +export type { Customize }; const fallback = (a?: string, b?: string): string => typeof a == "string" && a.length > 0 ? a : `${b}`; diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index e07369f0fb..4e2f0e416d 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -9,6 +9,7 @@ import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; import { parse_query } from "@cocalc/sync/table/util"; import sha1 from "sha1"; import { keys } from "lodash"; +import { type HubApi, initHubApi } from "@cocalc/nats/api/index"; export class NatsClient { /*private*/ client: WebappClient; @@ -18,9 +19,11 @@ export class NatsClient { // obviously just for learning: public nats = nats; public jetstream = jetstream; + public hub : HubApi; constructor(client: WebappClient) { this.client = client; + this.hub = initHubApi(this.callHubApi); } getConnection = reuseInFlight(async () => { @@ -44,27 +47,29 @@ export class NatsClient { return this.nc; }); - request = async (subject: string, data: string) => { - const c = await this.getConnection(); - const resp = await c.request(subject, this.sc.encode(data)); - return this.sc.decode(resp.data); - }; - - api = async ({ endpoint, params }: { endpoint: string; params?: object }) => { + private callHubApi = async ({ + name, + args, + }: { + name: string; + args: any[]; + }) => { const c = await this.getConnection(); const subject = `hub.account.api.${this.client.account_id}`; - console.log(`publishing to subject='${subject}'`); const resp = await c.request( subject, this.jc.encode({ - endpoint, - account_id: this.client.account_id, - params, + name, + args, }), ); - const x = this.jc.decode(resp.data); - console.log("got back ", x); - return x; + return this.jc.decode(resp.data); + }; + + request = async (subject: string, data: string) => { + const c = await this.getConnection(); + const resp = await c.request(subject, this.sc.encode(data)); + return this.sc.decode(resp.data); }; project = async ({ diff --git a/src/packages/nats/api/index.ts b/src/packages/nats/api/index.ts new file mode 100644 index 0000000000..24c053861a --- /dev/null +++ b/src/packages/nats/api/index.ts @@ -0,0 +1,18 @@ +import type { Customize } from "@cocalc/util/db-schema/server-settings"; + +export interface HubApi { + getCustomize: (fields?: string[]) => Promise; + userQuery: (opts: { + project_id?: string; + query: any; + options?: any[]; + }) => Promise; +} + +export function initHubApi(callHubApi): HubApi { + const hubApi: any = {}; + for (const name of ["getCustomize", "userQuery"]) { + hubApi[name] = async (...args) => await callHubApi({ name, args }); + } + return hubApi as HubApi; +} diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 0317f7cf66..0da1d46b89 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -3,7 +3,8 @@ "version": "1.0.0", "description": "CoCalc NATS integration code. Usable by both nodejs and browser.", "exports": { - "./sync/*": "./dist/sync/*.js" + "./sync/*": "./dist/sync/*.js", + "./api/*": "./dist/api/*.js" }, "scripts": { "preinstall": "npx only-allow pnpm", @@ -12,9 +13,17 @@ "test": "pnpm exec jest", "prepublishOnly": "pnpm test" }, - "files": ["dist/**", "README.md", "package.json"], + "files": [ + "dist/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["utilities", "nats", "cocalc"], + "keywords": [ + "utilities", + "nats", + "cocalc" + ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/nats": "workspace:*", diff --git a/src/packages/nats/tsconfig.json b/src/packages/nats/tsconfig.json index 98172f1f9b..61be42b5d0 100644 --- a/src/packages/nats/tsconfig.json +++ b/src/packages/nats/tsconfig.json @@ -4,5 +4,6 @@ "rootDir": "./", "outDir": "dist" }, - "exclude": ["node_modules", "dist", "test"] + "exclude": ["node_modules", "dist", "test"], + "references": [{ "path": "../database" }] } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 03e7d436d6..e3053c2376 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1414,6 +1414,9 @@ importers: '@cocalc/gcloud-pricing-calculator': specifier: ^1.13.0 version: 1.13.0 + '@cocalc/nats': + specifier: workspace:* + version: link:../nats '@cocalc/server': specifier: workspace:* version: 'link:' @@ -2070,7 +2073,7 @@ importers: version: 1.11.13 debug: specifier: ^4.4.0 - version: 4.4.0 + version: 4.4.0(supports-color@8.1.1) decimal.js-light: specifier: ^2.5.1 version: 2.5.1 @@ -13014,7 +13017,7 @@ snapshots: '@babel/traverse': 7.25.6 '@babel/types': 7.25.6 convert-source-map: 2.0.0 - debug: 4.4.0 + debug: 4.4.0(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -13557,7 +13560,7 @@ snapshots: '@babel/parser': 7.25.6 '@babel/template': 7.25.0 '@babel/types': 7.25.6 - debug: 4.4.0 + debug: 4.4.0(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -18059,10 +18062,6 @@ snapshots: dependencies: ms: 2.1.2 - debug@4.4.0: - dependencies: - ms: 2.1.3 - debug@4.4.0(supports-color@8.1.1): dependencies: ms: 2.1.3 @@ -20387,7 +20386,7 @@ snapshots: istanbul-lib-source-maps@4.0.1: dependencies: - debug: 4.4.0 + debug: 4.4.0(supports-color@8.1.1) istanbul-lib-coverage: 3.2.2 source-map: 0.6.1 transitivePeerDependencies: diff --git a/src/packages/server/nats/api.ts b/src/packages/server/nats/api.ts index e85161c893..4f24ea5f25 100644 --- a/src/packages/server/nats/api.ts +++ b/src/packages/server/nats/api.ts @@ -13,7 +13,7 @@ Optional: start more servers -- requests get randomly routed to exactly one of t To make use of this from a browser: - await cc.client.nats_client.api({endpoint:"customize", params:{fields:['siteName']}}) + await cc.client.nats_client.api({name:"customize", args:{fields:['siteName']}}) When you make changes, just restart the above. All clients will instantly use the new version after you restart, and there is no need to restart the hub @@ -27,6 +27,7 @@ To view all requests (and replies) in realtime: import { JSONCodec } from "nats"; import getLogger from "@cocalc/backend/logger"; import { isValidUUID } from "@cocalc/util/misc"; +import { type HubApi } from "@cocalc/nats/api/index"; const logger = getLogger("server:nats:api"); @@ -53,9 +54,9 @@ async function handleApiRequest(mesg) { } const request = jc.decode(mesg.data) ?? {}; // TODO: obviously user-provided account_id is no good! This is a POC. - const { endpoint, params } = request as any; - logger.debug("handling hub.api request:", { account_id, endpoint, params }); - resp = await getResponse(endpoint, account_id, params); + const { name, args } = request as any; + logger.debug("handling hub.api request:", { account_id, name, args }); + resp = await getResponse({ name, args, account_id }); } catch (err) { resp = { error: `${err}` }; } @@ -63,49 +64,26 @@ async function handleApiRequest(mesg) { } import userQuery from "@cocalc/database/user-query"; -import { execute as jupyterExecute } from "@cocalc/server/jupyter/execute"; -import getKernels from "@cocalc/server/jupyter/kernels"; import getCustomize from "@cocalc/database/settings/customize"; -import callProject from "@cocalc/server/projects/call"; -import isCollaborator from "@cocalc/server/projects/is-collaborator"; - -async function getResponse(endpoint, account_id, params) { - switch (endpoint) { - case "customize": - return await getCustomize(params?.fields); - case "user-query": - return { - query: await userQuery({ - ...params, - account_id, - }), - }; - case "exec": - if ( - !(await isCollaborator({ account_id, project_id: params.project_id })) - ) { - throw Error("user must be a collaborator on the project"); - } - return await callProject({ - account_id, - project_id: params.project_id, - mesg: { - event: "project_exec", - ...params, - }, - }); - case "jupyter/execute": - return { - ...(await jupyterExecute({ ...params, account_id })), - success: true, - }; - case "jupyter/kernels": - return { - ...(await getKernels({ ...params, account_id })), - success: true, - }; - - default: - throw Error(`unknown endpoint '${endpoint}'`); + +function getAccountId(args) { + return (args as any).account_id; +} + +const hubApi: HubApi = { + getCustomize, + userQuery: async (...args) => + await userQuery({ + ...args[0], + account_id: getAccountId(args), + }), +}; + +async function getResponse({ name, args, account_id }) { + const f = hubApi[name]; + if (f == null) { + throw Error(`unknown function '${name}'`); } + args.account_id = account_id; + return await f(args); } diff --git a/src/packages/server/package.json b/src/packages/server/package.json index 411176c38e..0ac34057dc 100644 --- a/src/packages/server/package.json +++ b/src/packages/server/package.json @@ -38,6 +38,7 @@ "@cocalc/backend": "workspace:*", "@cocalc/database": "workspace:*", "@cocalc/gcloud-pricing-calculator": "^1.13.0", + "@cocalc/nats": "workspace:*", "@cocalc/server": "workspace:*", "@cocalc/util": "workspace:*", "@google-ai/generativelanguage": "^2.7.0", diff --git a/src/packages/server/tsconfig.json b/src/packages/server/tsconfig.json index 5df79bdb81..d08ee56187 100644 --- a/src/packages/server/tsconfig.json +++ b/src/packages/server/tsconfig.json @@ -10,6 +10,7 @@ "references": [ { "path": "../backend" }, { "path": "../database" }, + { "path": "../nats" }, { "path": "../util" } ] } diff --git a/src/packages/util/db-schema/server-settings.ts b/src/packages/util/db-schema/server-settings.ts index 71c1bbdeff..a4480c0b1c 100644 --- a/src/packages/util/db-schema/server-settings.ts +++ b/src/packages/util/db-schema/server-settings.ts @@ -4,6 +4,8 @@ */ import { Table } from "./types"; +import type { KucalcValues } from "@cocalc/util/db-schema/site-defaults"; +import type { Strategy } from "@cocalc/util/types/sso"; Table({ name: "passport_settings", @@ -76,3 +78,54 @@ Table({ }, }, }); + +export interface Customize { + siteName?: string; + siteDescription?: string; + organizationName?: string; + organizationEmail?: string; + organizationURL?: string; + termsOfServiceURL?: string; + helpEmail?: string; + contactEmail?: string; + isCommercial?: boolean; + kucalc?: KucalcValues; + sshGateway?: boolean; + sshGatewayDNS?: string; + logoSquareURL?: string; + logoRectangularURL?: string; + splashImage?: string; + indexInfo?: string; + indexTagline?: string; + imprint?: string; + policies?: string; + shareServer?: boolean; + landingPages?: boolean; + dns?: string; + siteURL?: string; + googleAnalytics?: string; + anonymousSignup?: boolean; + anonymousSignupLicensedShares?: boolean; + emailSignup?: boolean; + accountCreationInstructions?: string; + zendesk?: boolean; // true if zendesk support is configured. + stripePublishableKey?: string; + imprint_html?: string; + policies_html?: string; + reCaptchaKey?: string; + sandboxProjectsEnabled?: boolean; + sandboxProjectId?: string; + verifyEmailAddresses?: boolean; + strategies?: Strategy[]; + openaiEnabled?: boolean; + googleVertexaiEnabled?: boolean; + mistralEnabled?: boolean; + anthropicEnabled?: boolean; + ollamaEnabled?: boolean; + neuralSearchEnabled?: boolean; + jupyterApiEnabled?: boolean; + computeServersEnabled?: boolean; + cloudFilesystemsEnabled?: boolean; + githubProjectId?: string; + support?: string; +} From aa2271817e9a85eacb71b4068f2b0f749cdc371a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 26 Jan 2025 03:34:13 +0000 Subject: [PATCH 039/281] nats: starting to add db service --- docs/nats/devlog.md | 24 ++++++++- src/packages/backend/nats/index.ts | 33 +++++++++++++ src/packages/backend/package.json | 14 ++---- src/packages/database/nats/index.ts | 73 ++++++++++++++++++++++++++++ src/packages/database/package.json | 2 + src/packages/frontend/client/nats.ts | 13 ++--- src/packages/pnpm-lock.yaml | 7 +++ src/packages/server/nats/api.ts | 28 +++++------ src/packages/server/nats/auth.ts | 8 ++- src/packages/server/nats/index.ts | 21 +------- 10 files changed, 171 insertions(+), 52 deletions(-) create mode 100644 src/packages/backend/nats/index.ts create mode 100644 src/packages/database/nats/index.ts diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index cf77b5e794..86d0b95cfb 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -249,7 +249,29 @@ Plan. - [ ] Stream: Records everything with this subject `project.${project_id}.patches` - [ ] It would be very nice if we can use the server assigned timestamps.... but probably not - [ ] For transitioning and de\-archiving, there must be a way to do this, since they have a backup/restore process -- + +## [ ] Goal: PostgreSQL Changefeed Synctable + +This is critical to solve. This sucks now. This is key to eliminating "hub\-websocket". This might be very easy. Here's the plan: + +- make a request/response listener that listens on hub.account.{account\_id} and hub.db.project.{project\_id} for a db query. +- if changes is false, just responds with the result of the query. +- if changes is true, get kv store k named `account-{account_id}` or `project-{project_id}` \(which might be used by project or compute server\). + - let id be the sha1 hash of the query \(and options\) + - k.id.update is less than X seconds ago, do nothing... it's already being updated by another server. + - do the query to the database \(with changes true\) + - write the results into k under k.id.data.key = value. + - keep watching for changes so long as k.id.interest is at most n\*X seconds ago. + - Also set k.id.update to now. + - return id +- another message to `hub.db.{account_id}` which contains a list of id's. + - When get this one, update k.id.interest to now for each of the id's. + +With the above algorithm, it should be very easy to reimplement the client side of SyncTable. Moreover, there are many advantages: + +- For a fixed account\_id or project\-id, there's no extra work at all for 1 versus 100 of them. I.e., this is great for opening a bunch of distinct browser windows. +- If you refresh your browser, everything stays stable \-\- nothing changes at all and you instantly have your data. Same if the network drops and resumes. +- When implementing our new synctable, we can immediately start with the possibly stale data from the last time it was active, then update it to the correct data. Thus even if everything but NATS is done/unavailable, the experience would be much better. It's like "local first", but somehow "network mesh first". With a leaf node it would literally be local first. ## [ ] Goal: Terminal and **compute server** diff --git a/src/packages/backend/nats/index.ts b/src/packages/backend/nats/index.ts new file mode 100644 index 0000000000..622e3d486a --- /dev/null +++ b/src/packages/backend/nats/index.ts @@ -0,0 +1,33 @@ +import { join } from "path"; +import { secrets } from "@cocalc/backend/data"; +import { readFile } from "node:fs/promises"; +import getLogger from "@cocalc/backend/logger"; +import { connect, credsAuthenticator } from "nats"; + +const logger = getLogger("backend:nats"); + +export async function getCreds(): Promise { + const filename = join(secrets, "nats.creds"); + try { + return (await readFile(filename)).toString().trim(); + } catch { + logger.debug( + `getCreds -- please create ${filename}, which is missing. Nothing will work.`, + ); + return undefined; + } +} + +let nc: Awaited> | null = null; +export async function getConnection() { + logger.debug("connecting to nats"); + + if (nc == null) { + const creds = await getCreds(); + nc = await connect({ + authenticator: credsAuthenticator(new TextEncoder().encode(creds)), + }); + logger.debug(`connected to ${nc.getServer()}`); + } + return nc; +} diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 0cb3d239ef..67ae23cfd2 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -5,14 +5,12 @@ "exports": { "./*": "./dist/*.js", "./database": "./dist/database/index.js", + "./nats": "./dist/nats/index.js", "./server-settings": "./dist/server-settings/index.js", "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": [ - "utilities", - "cocalc" - ], + "keywords": ["utilities", "cocalc"], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -21,12 +19,7 @@ "test": "pnpm exec jest --detectOpenHandles", "prepublishOnly": "pnpm test" }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { @@ -40,6 +33,7 @@ "fs-extra": "^11.2.0", "lodash": "^4.17.21", "lru-cache": "^7.18.3", + "nats": "^2.29.1", "password-hash": "^1.2.2", "prom-client": "^13.0.0", "rimraf": "^5.0.5", diff --git a/src/packages/database/nats/index.ts b/src/packages/database/nats/index.ts new file mode 100644 index 0000000000..2eb676a07c --- /dev/null +++ b/src/packages/database/nats/index.ts @@ -0,0 +1,73 @@ +/* + + + echo "require('@cocalc/database/nats').init()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node + +*/ + +import getLogger from "@cocalc/backend/logger"; +import { JSONCodec } from "nats"; +import { isValidUUID } from "@cocalc/util/misc"; +import userQuery from "@cocalc/database/user-query"; +import { getConnection } from "@cocalc/backend/nats"; + +const logger = getLogger("database:nats"); + +const jc = JSONCodec(); + +export async function init() { + const subject = "hub.*.*.db"; + logger.debug(`init -- subject='${subject}', options=`, { + queue: "0", + }); + const nc = await getConnection(); + const sub = nc.subscribe(subject, { queue: "0" }); + for await (const mesg of sub) { + handleRequest(mesg); + } +} + +async function handleRequest(mesg) { + console.log({ subject: mesg.subject }); + let resp; + try { + const segments = mesg.subject.split("."); + const uuid = segments[2]; + if (!isValidUUID(uuid)) { + throw Error(`invalid uuid '${uuid}'`); + } + const type = segments[1]; // 'project' or 'account' + let account_id, project_id; + if (type == "project") { + project_id = uuid; + account_id = undefined; + } else if (type == "account") { + project_id = undefined; + account_id = uuid; + } else { + throw Error("must be project or account"); + } + const { name, args } = jc.decode(mesg.data) ?? ({} as any); + if (!name) { + throw Error("api endpoint name must be given in message"); + } + logger.debug("handling hub db request:", { + account_id, + project_id, + name, + args, + }); + resp = await getResponse({ name, args, account_id, project_id }); + } catch (err) { + resp = { error: `${err}` }; + } + mesg.respond(jc.encode(resp)); +} + +async function getResponse({ name, args, account_id, project_id }) { + if (name == "userQuery") { + return await userQuery({ ...args[0], account_id, project_id }); + } else { + throw Error(`name='${name}' not implemented`); + } +} diff --git a/src/packages/database/package.json b/src/packages/database/package.json index 8055b6a727..695bfcf71a 100644 --- a/src/packages/database/package.json +++ b/src/packages/database/package.json @@ -5,6 +5,7 @@ "exports": { ".": "./dist/index.js", "./accounts/*": "./dist/accounts/*.js", + "./nats": "./dist/nats/index.js", "./pool": "./dist/pool/index.js", "./pool/*": "./dist/pool/*.js", "./postgres/*": "./dist/postgres/*.js", @@ -31,6 +32,7 @@ "json-stable-stringify": "^1.0.1", "lodash": "^4.17.21", "lru-cache": "^7.18.3", + "nats": "^2.29.1", "node-fetch": "2.6.7", "pg": "^8.7.1", "random-key": "^0.3.2", diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 4e2f0e416d..2d179a41d7 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -16,14 +16,13 @@ export class NatsClient { private sc = nats.StringCodec(); private jc = nats.JSONCodec(); private nc?: Awaited>; - // obviously just for learning: public nats = nats; public jetstream = jetstream; - public hub : HubApi; + public hub: HubApi; constructor(client: WebappClient) { this.client = client; - this.hub = initHubApi(this.callHubApi); + this.hub = initHubApi(this.callHub); } getConnection = reuseInFlight(async () => { @@ -47,15 +46,17 @@ export class NatsClient { return this.nc; }); - private callHubApi = async ({ + private callHub = async ({ + service = "api", name, - args, + args = [], }: { + service?: string; name: string; args: any[]; }) => { const c = await this.getConnection(); - const subject = `hub.account.api.${this.client.account_id}`; + const subject = `hub.account.${this.client.account_id}.${service}`; const resp = await c.request( subject, this.jc.encode({ diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index e3053c2376..0df8a5464a 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -103,6 +103,9 @@ importers: lru-cache: specifier: ^7.18.3 version: 7.18.3 + nats: + specifier: ^2.29.1 + version: 2.29.1 password-hash: specifier: ^1.2.2 version: 1.2.2 @@ -204,6 +207,9 @@ importers: lru-cache: specifier: ^7.18.3 version: 7.18.3 + nats: + specifier: ^2.29.1 + version: 2.29.1 node-fetch: specifier: 2.6.7 version: 2.6.7(encoding@0.1.13) @@ -9108,6 +9114,7 @@ packages: lodash.get@4.4.2: resolution: {integrity: sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==} + deprecated: This package is deprecated. Use the optional chaining (?.) operator instead. lodash.includes@4.3.0: resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} diff --git a/src/packages/server/nats/api.ts b/src/packages/server/nats/api.ts index 4f24ea5f25..39f44127b7 100644 --- a/src/packages/server/nats/api.ts +++ b/src/packages/server/nats/api.ts @@ -34,10 +34,11 @@ const logger = getLogger("server:nats:api"); const jc = JSONCodec(); export async function initAPI(nc) { - logger.debug("initAPI -- subject='hub.account.api', options=", { + const subject = "hub.*.*.api"; + logger.debug(`initAPI -- subject='${subject}', options=`, { queue: "0", }); - const sub = nc.subscribe("hub.account.api.>", { queue: "0" }); + const sub = nc.subscribe(subject, { queue: "0" }); for await (const mesg of sub) { handleApiRequest(mesg); } @@ -48,12 +49,15 @@ async function handleApiRequest(mesg) { let resp; try { const segments = mesg.subject.split("."); - const account_id = segments[3]; + const type = segments[1]; + if (type != "account") { + throw Error("only type='account' is supported now"); + } + const account_id = segments[2]; if (!isValidUUID(account_id)) { throw Error(`invalid account_id '${account_id}'`); } const request = jc.decode(mesg.data) ?? {}; - // TODO: obviously user-provided account_id is no good! This is a POC. const { name, args } = request as any; logger.debug("handling hub.api request:", { account_id, name, args }); resp = await getResponse({ name, args, account_id }); @@ -66,17 +70,9 @@ async function handleApiRequest(mesg) { import userQuery from "@cocalc/database/user-query"; import getCustomize from "@cocalc/database/settings/customize"; -function getAccountId(args) { - return (args as any).account_id; -} - const hubApi: HubApi = { getCustomize, - userQuery: async (...args) => - await userQuery({ - ...args[0], - account_id: getAccountId(args), - }), + userQuery, }; async function getResponse({ name, args, account_id }) { @@ -84,6 +80,8 @@ async function getResponse({ name, args, account_id }) { if (f == null) { throw Error(`unknown function '${name}'`); } - args.account_id = account_id; - return await f(args); + if (name == "userQuery" && args[0] != null) { + args[0].account_id = account_id; + } + return await f(...args); } diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 5a1fcdd46b..b02c6f2e7a 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -15,6 +15,12 @@ Points that took me a while to figure out: DOCS: - https://nats-io.github.io/nsc/ + +USAGE: + +a = require('@cocalc/server/nats/auth') +await a.configureNatsUser({account_id:'275f1db7-bf37-4b44-b9aa-d64694269c9f'}) +await a.configureNatsUser({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}) */ import { executeCode } from "@cocalc/backend/execute-code"; @@ -110,7 +116,7 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { throw Error("must be a valid uuid"); } const userType = getCoCalcUserType(cocalcUser); - const goalPub = new Set(["_INBOX.>", `hub.${userType}.api.${userId}`]); + const goalPub = new Set(["_INBOX.>", `hub.${userType}.${userId}.>`]); const goalSub = new Set(["_INBOX.>"]); if (userType == "account") { diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts index 5d7cd32508..1f5ea2047e 100644 --- a/src/packages/server/nats/index.ts +++ b/src/packages/server/nats/index.ts @@ -1,29 +1,12 @@ -import { connect, credsAuthenticator } from "nats"; import getLogger from "@cocalc/backend/logger"; import { initAPI } from "./api"; +import { getConnection } from "@cocalc/backend/nats"; const logger = getLogger("server:nats"); -const creds = `-----BEGIN NATS USER JWT----- -eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJVTDJaWEdFWFFKTzVRNjdLU1hKNDdERFpKSFE3QUFWMjdHWUtBN1ZJVjVaT01DQU1SN1hBIiwiaWF0IjoxNzM3NTY3OTQwLCJpc3MiOiJBRDRHNlI2MkJERFFVU0NKVkxaTkE3RVM3UjNBNkRXWExZVVdHWlY3NEVKMlM2VkJDN0RRVk0zSSIsIm5hbWUiOiJhZG1pbiIsInN1YiI6IlVBV1hZVUpYSEFXQzNPSFFURE1SQVBSWVpNNFQ0RkZDRk1TTVFLNDVCWU1SS0ZSRE5RTjQ0Vk1SIiwibmF0cyI6eyJwdWIiOnsiYWxsb3ciOlsiXHUwMDNlIl19LCJzdWIiOnsiYWxsb3ciOlsiXHUwMDNlIl19LCJzdWJzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJ0eXBlIjoidXNlciIsInZlcnNpb24iOjJ9fQ.Pv9-T3P7cO1VSFiNocGA0vCGvwQ-UaX3b7OzwMIHdn5hGs4kUv4eLE-Er_6dxrZiPu6PJjBYB7eD2hyb-gxSCQ -------END NATS USER JWT------ - -************************* IMPORTANT ************************* -NKEY Seed printed below can be used to sign and prove identity. -NKEYs are sensitive and should be treated as secrets. - ------BEGIN USER NKEY SEED----- -SUAMW6S2OXSKL2ETX5GJE3NDLWGXZFZ4JAP5WHBCK43RMFDPJCCJLPWC5Y -------END USER NKEY SEED------ - -************************************************************* -`; - export default async function initNatsServer() { logger.debug("initializing nats cocalc hub server"); - const nc = await connect({ - authenticator: credsAuthenticator(new TextEncoder().encode(creds)), - }); + const nc = await getConnection(); logger.debug(`connected to ${nc.getServer()}`); initAPI(nc); } From b0b36bffb4a4245e4d0b85b56959697a4172d18d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 26 Jan 2025 04:31:19 +0000 Subject: [PATCH 040/281] nats api: a little bit of refactoring --- docs/nats/devlog.md | 8 ++++---- src/packages/database/nats/index.ts | 19 ++---------------- src/packages/database/package.json | 1 + src/packages/database/tsconfig.json | 3 ++- src/packages/frontend/client/nats.ts | 1 + src/packages/nats/api/index.ts | 26 ++++++++++++++++++++++++ src/packages/nats/package.json | 13 +++--------- src/packages/pnpm-lock.yaml | 3 +++ src/packages/server/nats/api.ts | 30 ++++++++++++++-------------- src/packages/server/nats/auth.ts | 3 +++ 10 files changed, 60 insertions(+), 47 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 86d0b95cfb..6030381fe4 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -254,9 +254,9 @@ Plan. This is critical to solve. This sucks now. This is key to eliminating "hub\-websocket". This might be very easy. Here's the plan: -- make a request/response listener that listens on hub.account.{account\_id} and hub.db.project.{project\_id} for a db query. -- if changes is false, just responds with the result of the query. -- if changes is true, get kv store k named `account-{account_id}` or `project-{project_id}` \(which might be used by project or compute server\). +- [x] make a request/response listener that listens on hub.account.{account\_id} and hub.db.project.{project\_id} for a db query. +- [x] if changes is false, just responds with the result of the query. +- [ ] if changes is true, get kv store k named `account-{account_id}` or `project-{project_id}` \(which can be used by project or compute server\). - let id be the sha1 hash of the query \(and options\) - k.id.update is less than X seconds ago, do nothing... it's already being updated by another server. - do the query to the database \(with changes true\) @@ -264,7 +264,7 @@ This is critical to solve. This sucks now. This is key to eliminating "hub\-w - keep watching for changes so long as k.id.interest is at most n\*X seconds ago. - Also set k.id.update to now. - return id -- another message to `hub.db.{account_id}` which contains a list of id's. +- [ ] another message to `hub.db.{account_id}` which contains a list of id's. - When get this one, update k.id.interest to now for each of the id's. With the above algorithm, it should be very easy to reimplement the client side of SyncTable. Moreover, there are many advantages: diff --git a/src/packages/database/nats/index.ts b/src/packages/database/nats/index.ts index 2eb676a07c..7648c499bc 100644 --- a/src/packages/database/nats/index.ts +++ b/src/packages/database/nats/index.ts @@ -7,9 +7,9 @@ import getLogger from "@cocalc/backend/logger"; import { JSONCodec } from "nats"; -import { isValidUUID } from "@cocalc/util/misc"; import userQuery from "@cocalc/database/user-query"; import { getConnection } from "@cocalc/backend/nats"; +import { getUserId } from "@cocalc/nats/api"; const logger = getLogger("database:nats"); @@ -31,22 +31,7 @@ async function handleRequest(mesg) { console.log({ subject: mesg.subject }); let resp; try { - const segments = mesg.subject.split("."); - const uuid = segments[2]; - if (!isValidUUID(uuid)) { - throw Error(`invalid uuid '${uuid}'`); - } - const type = segments[1]; // 'project' or 'account' - let account_id, project_id; - if (type == "project") { - project_id = uuid; - account_id = undefined; - } else if (type == "account") { - project_id = undefined; - account_id = uuid; - } else { - throw Error("must be project or account"); - } + const { account_id, project_id } = getUserId(mesg.subject); const { name, args } = jc.decode(mesg.data) ?? ({} as any); if (!name) { throw Error("api endpoint name must be given in message"); diff --git a/src/packages/database/package.json b/src/packages/database/package.json index 695bfcf71a..368aef5dac 100644 --- a/src/packages/database/package.json +++ b/src/packages/database/package.json @@ -21,6 +21,7 @@ "@cocalc/backend": "workspace:*", "@cocalc/database": "workspace:*", "@cocalc/util": "workspace:*", + "@cocalc/nats": "workspace:*", "@types/lodash": "^4.14.202", "@types/pg": "^8.6.1", "@types/uuid": "^8.3.1", diff --git a/src/packages/database/tsconfig.json b/src/packages/database/tsconfig.json index 02282074ad..a9234ba729 100644 --- a/src/packages/database/tsconfig.json +++ b/src/packages/database/tsconfig.json @@ -7,7 +7,8 @@ }, "exclude": ["node_modules", "dist"], "references": [ - { "path": "../util" }, { "path": "../backend" }, + { "path": "../nats" }, + { "path": "../util" } ] } diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 2d179a41d7..2b3dface10 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -63,6 +63,7 @@ export class NatsClient { name, args, }), + { timeout: 5000 }, ); return this.jc.decode(resp.data); }; diff --git a/src/packages/nats/api/index.ts b/src/packages/nats/api/index.ts index 24c053861a..263ba095bf 100644 --- a/src/packages/nats/api/index.ts +++ b/src/packages/nats/api/index.ts @@ -1,4 +1,5 @@ import type { Customize } from "@cocalc/util/db-schema/server-settings"; +import { isValidUUID } from "@cocalc/util/misc"; export interface HubApi { getCustomize: (fields?: string[]) => Promise; @@ -16,3 +17,28 @@ export function initHubApi(callHubApi): HubApi { } return hubApi as HubApi; } + +type UserId = + | { + account_id: string; + project_id: undefined; + } + | { + account_id: undefined; + project_id: string; + }; +export function getUserId(subject: string): UserId { + const segments = subject.split("."); + const uuid = segments[2]; + if (!isValidUUID(uuid)) { + throw Error(`invalid uuid '${uuid}'`); + } + const type = segments[1]; // 'project' or 'account' + if (type == "project") { + return { project_id: uuid } as UserId; + } else if (type == "account") { + return { account_id: uuid } as UserId; + } else { + throw Error("must be project or account"); + } +} diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 0da1d46b89..778d8f24c5 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -4,6 +4,7 @@ "description": "CoCalc NATS integration code. Usable by both nodejs and browser.", "exports": { "./sync/*": "./dist/sync/*.js", + "./api": "./dist/api/index.js", "./api/*": "./dist/api/*.js" }, "scripts": { @@ -13,17 +14,9 @@ "test": "pnpm exec jest", "prepublishOnly": "pnpm test" }, - "files": [ - "dist/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "nats", - "cocalc" - ], + "keywords": ["utilities", "nats", "cocalc"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/nats": "workspace:*", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 0df8a5464a..53f79680da 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -171,6 +171,9 @@ importers: '@cocalc/database': specifier: workspace:* version: 'link:' + '@cocalc/nats': + specifier: workspace:* + version: link:../nats '@cocalc/util': specifier: workspace:* version: link:../util diff --git a/src/packages/server/nats/api.ts b/src/packages/server/nats/api.ts index 39f44127b7..8cdac3d658 100644 --- a/src/packages/server/nats/api.ts +++ b/src/packages/server/nats/api.ts @@ -26,8 +26,7 @@ To view all requests (and replies) in realtime: import { JSONCodec } from "nats"; import getLogger from "@cocalc/backend/logger"; -import { isValidUUID } from "@cocalc/util/misc"; -import { type HubApi } from "@cocalc/nats/api/index"; +import { type HubApi, getUserId } from "@cocalc/nats/api/index"; const logger = getLogger("server:nats:api"); @@ -48,19 +47,16 @@ async function handleApiRequest(mesg) { console.log({ subject: mesg.subject }); let resp; try { - const segments = mesg.subject.split("."); - const type = segments[1]; - if (type != "account") { - throw Error("only type='account' is supported now"); - } - const account_id = segments[2]; - if (!isValidUUID(account_id)) { - throw Error(`invalid account_id '${account_id}'`); - } + const { account_id, project_id } = getUserId(mesg.subject); const request = jc.decode(mesg.data) ?? {}; const { name, args } = request as any; - logger.debug("handling hub.api request:", { account_id, name, args }); - resp = await getResponse({ name, args, account_id }); + logger.debug("handling hub.api request:", { + account_id, + project_id, + name, + args, + }); + resp = await getResponse({ name, args, account_id, project_id }); } catch (err) { resp = { error: `${err}` }; } @@ -75,13 +71,17 @@ const hubApi: HubApi = { userQuery, }; -async function getResponse({ name, args, account_id }) { +async function getResponse({ name, args, account_id, project_id }) { const f = hubApi[name]; if (f == null) { throw Error(`unknown function '${name}'`); } if (name == "userQuery" && args[0] != null) { - args[0].account_id = account_id; + if (account_id) { + args[0].account_id = account_id; + } else if (project_id) { + args[0].project_id = project_id; + } } return await f(...args); } diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index b02c6f2e7a..3eb51b7c76 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -120,6 +120,9 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { const goalSub = new Set(["_INBOX.>"]); if (userType == "account") { + + goalSub.add(`$KV.account-${userId}.>`); + const pool = getPool(); // all RUNNING projects with the user's group const query = `SELECT project_id, users#>>'{${userId},group}' AS group FROM projects WHERE state#>>'{state}'='running' AND users ? '${userId}' ORDER BY project_id`; From 5315d47be904f85638e9f474c861416856edc8be Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 26 Jan 2025 05:40:44 +0000 Subject: [PATCH 041/281] nats: foundations for changefeeds --- src/packages/database/nats/index.ts | 63 ++++++++++++++++++++-- src/packages/database/package.json | 2 +- src/packages/frontend/client/nats.ts | 1 + src/packages/nats/sync/synctable-kv.ts | 61 +++++++++++++++++---- src/packages/nats/sync/synctable-stream.ts | 21 ++++++-- src/packages/nats/sync/synctable.ts | 16 ++++-- src/packages/pnpm-lock.yaml | 6 ++- 7 files changed, 146 insertions(+), 24 deletions(-) diff --git a/src/packages/database/nats/index.ts b/src/packages/database/nats/index.ts index 7648c499bc..400befb1a2 100644 --- a/src/packages/database/nats/index.ts +++ b/src/packages/database/nats/index.ts @@ -10,6 +10,10 @@ import { JSONCodec } from "nats"; import userQuery from "@cocalc/database/user-query"; import { getConnection } from "@cocalc/backend/nats"; import { getUserId } from "@cocalc/nats/api"; +import { callback } from "awaiting"; +import { db } from "@cocalc/database"; +import { SyncTableKV } from "@cocalc/nats/sync/synctable-kv"; +import { sha1 } from "@cocalc/backend/misc_node"; const logger = getLogger("database:nats"); @@ -23,11 +27,11 @@ export async function init() { const nc = await getConnection(); const sub = nc.subscribe(subject, { queue: "0" }); for await (const mesg of sub) { - handleRequest(mesg); + handleRequest(mesg, nc); } } -async function handleRequest(mesg) { +async function handleRequest(mesg, nc) { console.log({ subject: mesg.subject }); let resp; try { @@ -42,17 +46,66 @@ async function handleRequest(mesg) { name, args, }); - resp = await getResponse({ name, args, account_id, project_id }); + resp = await getResponse({ name, args, account_id, project_id, nc }); } catch (err) { resp = { error: `${err}` }; } mesg.respond(jc.encode(resp)); } -async function getResponse({ name, args, account_id, project_id }) { +async function getResponse({ name, args, account_id, project_id, nc }) { if (name == "userQuery") { - return await userQuery({ ...args[0], account_id, project_id }); + const opts = { ...args[0], account_id, project_id }; + if (opts.changes == null) { + return await userQuery(opts); + } else { + return await createChangefeed(opts, nc); + } } else { throw Error(`name='${name}' not implemented`); } } + +// This is tricky. We return the first result as a normal +// async function, but then handle (and don't return) +// the subsequent calls to cb generated by the changefeed. +async function createChangefeed(opts, nc) { + const query = opts.query; + const env = { nc, jc, sha1 }; + const synctable = new SyncTableKV({ + query, + env, + account_id: opts.account_id, + project_id: opts.project_id, + }); + await synctable.init(); + const f = (cb) => { + let first = true; + db().user_query({ + ...opts, + cb: async (err, result) => { + if (first) { + first = false; + cb(err, result); + if (result != null) { + for (const x of result[synctable.table]) { + logger.debug("changefeed init", x); + await synctable.set(x); + } + } + return; + } + logger.debug("changefeed", result); + const { action, new_val, old_val } = result as any; + // action = 'insert', 'update', 'delete', 'close' + // e.g., {"action":"insert","new_val":{"title":"testingxxxxx","project_id":"81e0c408-ac65-4114-bad5-5f4b6539bd0e"}} + if (action == "insert" || action == "update") { + await synctable.set(new_val); + } else if (action == "delete") { + await synctable.delete(old_val); + } + }, + }); + }; + return await callback(f); +} diff --git a/src/packages/database/package.json b/src/packages/database/package.json index 368aef5dac..1e5c4f60a1 100644 --- a/src/packages/database/package.json +++ b/src/packages/database/package.json @@ -20,8 +20,8 @@ "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/database": "workspace:*", - "@cocalc/util": "workspace:*", "@cocalc/nats": "workspace:*", + "@cocalc/util": "workspace:*", "@types/lodash": "^4.14.202", "@types/pg": "^8.6.1", "@types/uuid": "^8.3.1", diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 2b3dface10..d134d5fcf0 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -133,6 +133,7 @@ export class NatsClient { jc: this.jc, nc: await this.getConnection(), }, + account_id: this.client.account_id, }); await s.init(); return s; diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index c1012b58a1..d862bc1790 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -12,12 +12,30 @@ import { Kvm } from "@nats-io/kv"; import sha1 from "sha1"; import jsonStableStringify from "json-stable-stringify"; import { keys } from "lodash"; -import { isValidUUID } from "@cocalc/util/misc"; import { client_db } from "@cocalc/util/db-schema/client-db"; -export async function getKv({ nc, project_id }) { +export async function getKv({ + nc, + project_id, + account_id, +}: { + nc; + project_id?: string; + account_id?: string; +}) { + if (account_id && project_id) { + throw Error("both account_id and project_id can't be defined"); + } + let name; + if (account_id) { + name = `account-${account_id}`; + } else if (project_id) { + name = `project-${project_id}`; + } else { + throw Error("one of account_id or project_id must be defined"); + } const kvm = new Kvm(nc); - return await kvm.create(`project-${project_id}`, { compression: true }); + return await kvm.create(name, { compression: true }); } interface NatsEnv { @@ -42,22 +60,31 @@ export class SyncTableKV { private nc; private jc; private sha1; - private table; + public readonly table; private primaryKeys: string[]; private primaryKeysSet: Set; private fields: string[]; - private project_id: string; + private project_id?: string; + private account_id?: string; - constructor({ query, env }: { query; env: NatsEnv }) { + constructor({ + query, + env, + account_id, + project_id, + }: { + query; + env: NatsEnv; + account_id?: string; + project_id?: string; + }) { this.sha1 = env.sha1 ?? sha1; this.nc = env.nc; this.jc = env.jc; const table = keys(query)[0]; this.table = table; - this.project_id = query[table][0].project_id; - if (!isValidUUID(this.project_id)) { - throw Error("query MUST specify a valid project_id"); - } + this.project_id = project_id ?? query[table][0].project_id; + this.account_id = account_id ?? query[table][0].account_id; this.primaryKeys = client_db.primary_keys(table); this.primaryKeysSet = new Set(this.primaryKeys); this.fields = keys(query[table][0]).filter( @@ -66,7 +93,11 @@ export class SyncTableKV { } init = async () => { - this.kv = await getKv({ nc: this.nc, project_id: this.project_id }); + this.kv = await getKv({ + nc: this.nc, + project_id: this.project_id, + account_id: this.account_id, + }); }; private primaryString = (obj): string => { @@ -102,6 +133,14 @@ export class SyncTableKV { } }; + delete = async (obj) => { + const key = this.getKey(obj); + const keys = await this.kv.keys(`${key}.>`); + for await (const k of keys) { + await this.kv.delete(k); + } + }; + get = async (obj?, field?) => { if (obj == null) { // everything known in this table by the project diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts index 242e0da978..29a259533b 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/nats/sync/synctable-stream.ts @@ -42,7 +42,8 @@ export class SyncTableStream extends EventEmitter { private sha1; private table; private primaryKeys: string[]; - private project_id: string; + private project_id?: string; + private account_id?: string; private streamName: string; private streamSubject: string; private path: string; @@ -52,7 +53,17 @@ export class SyncTableStream extends EventEmitter { private consumer?; private state: State = "disconnected"; - constructor({ query, env }: { query; env: NatsEnv }) { + constructor({ + query, + env, + account_id, + project_id, + }: { + query; + env: NatsEnv; + account_id?: string; + project_id?: string; + }) { super(); this.sha1 = env.sha1 ?? sha1; this.nc = env.nc; @@ -62,10 +73,14 @@ export class SyncTableStream extends EventEmitter { if (table != "patches") { throw Error("only the patches table is supported"); } - this.project_id = query[table][0].project_id; + this.project_id = project_id ?? query[table][0].project_id; + this.account_id = account_id ?? query[table][0].account_id; if (!isValidUUID(this.project_id)) { throw Error("query MUST specify a valid project_id"); } + if (this.account_id && !isValidUUID(this.account_id)) { + throw Error("query MUST specify a valid account_id"); + } this.path = query[table][0].path; if (!this.path) { throw Error("path MUST be specified"); diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/nats/sync/synctable.ts index fff6edc9ec..c8d90f8a73 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/nats/sync/synctable.ts @@ -4,10 +4,20 @@ import { keys } from "lodash"; export type SyncTable = SyncTableKV | SyncTableStream; -export function createSyncTable({ query, env }) { +export function createSyncTable({ + query, + env, + account_id, + project_id, +}: { + query; + env; + account_id?; + project_id?; +}) { const table = keys(query)[0]; if (table == "patches") { - return new SyncTableStream({ query, env }); + return new SyncTableStream({ query, env, account_id, project_id }); } - return new SyncTableKV({ query, env }); + return new SyncTableKV({ query, env, account_id, project_id }); } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 53f79680da..2748115531 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -197,7 +197,7 @@ importers: version: 8.7.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0 immutable: specifier: ^4.3.0 version: 4.3.7 @@ -18072,6 +18072,10 @@ snapshots: dependencies: ms: 2.1.2 + debug@4.4.0: + dependencies: + ms: 2.1.3 + debug@4.4.0(supports-color@8.1.1): dependencies: ms: 2.1.3 From 08e80a8a93c0dd708c63692f97042287ccb51c48 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 26 Jan 2025 06:02:10 +0000 Subject: [PATCH 042/281] nats: organizing startup/config --- src/package.json | 3 ++- src/packages/nats/sync/synctable-kv.ts | 7 ++++++ src/scripts/nats.conf | 33 ++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 src/scripts/nats.conf diff --git a/src/package.json b/src/package.json index b3600a4773..a855853d1b 100644 --- a/src/package.json +++ b/src/package.json @@ -17,7 +17,8 @@ "version-check": "pip3 install typing_extensions mypy || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py", "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test", "test": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r test", - "prettier-all": "cd packages/" + "prettier-all": "cd packages/", + "nats-server": "nats-server -c ./scripts/nats.conf" }, "repository": { "type": "git", diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index d862bc1790..80b9ed1b59 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -143,7 +143,14 @@ export class SyncTableKV { get = async (obj?, field?) => { if (obj == null) { + // everything known in this table by the project + // TODO: this is way to broad -- we probably need to restructure things + // by adding in a hash of the query? e.g., {this.table}.{hash(query)}.(same as before), + // since there's many queries for one table and below gives too much and we can't client + // side do all the filters...? or shouldn't (too slow). But also worry about clutter + // and maybe ttl? + const keys = await this.kv.keys(`${this.table}.>`); const all: any = {}; for await (const key of keys) { diff --git a/src/scripts/nats.conf b/src/scripts/nats.conf new file mode 100644 index 0000000000..e1dee3b6fb --- /dev/null +++ b/src/scripts/nats.conf @@ -0,0 +1,33 @@ +jetstream: enabled + +jetstream { + store_dir: data/nats/jetstream +} + +websocket { + listen: "localhost:8443" + no_tls: true + jwt_cookie: "%2F3fa218e5-7196-4020-8b30-e2127847cc4f%2Fport%2F5002cocalc_nats_jwt_cookie" +} + +include ../data/nats/trust.conf + +# configuration of the nats based resolver +resolver { + type: full + # Directory in which the account jwt will be stored + dir: 'data/nats/jwt' + # In order to support jwt deletion, set to true + # If the resolver type is full delete will rename the jwt. + # This is to allow manual restoration in case of inadvertent deletion. + # To restore a jwt, remove the added suffix .delete and restart or send a reload signal. + # To free up storage you must manually delete files with the suffix .delete. + allow_delete: false + # Interval at which a nats-server with a nats based account resolver will compare + # it's state with one random nats based account resolver in the cluster and if needed, + # exchange jwt and converge on the same set of jwt. + interval: "2m" + # Timeout for lookup requests in case an account does not exist locally. + timeout: "1.9s" +} + From ea1669be1ddec232f31d3ee2e3d66ad258a51b6b Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 26 Jan 2025 18:33:20 +0000 Subject: [PATCH 043/281] nats synctable: added support for atomic tables, which we'll use for database-backed changefeeds served by the hub --- src/packages/database/nats/index.ts | 15 ++- src/packages/frontend/client/nats.ts | 33 +++++- src/packages/nats/sync/synctable-kv-atomic.ts | 105 ++++++++++++++++++ src/packages/nats/sync/synctable-kv.ts | 27 +++-- src/packages/nats/sync/synctable-stream.ts | 6 +- src/packages/nats/sync/synctable.ts | 28 +++-- src/packages/project/client.ts | 4 +- src/packages/project/nats/synctable.ts | 35 +++--- src/packages/sync/editor/generic/sync-doc.ts | 1 + 9 files changed, 211 insertions(+), 43 deletions(-) create mode 100644 src/packages/nats/sync/synctable-kv-atomic.ts diff --git a/src/packages/database/nats/index.ts b/src/packages/database/nats/index.ts index 400befb1a2..604f767d04 100644 --- a/src/packages/database/nats/index.ts +++ b/src/packages/database/nats/index.ts @@ -12,7 +12,7 @@ import { getConnection } from "@cocalc/backend/nats"; import { getUserId } from "@cocalc/nats/api"; import { callback } from "awaiting"; import { db } from "@cocalc/database"; -import { SyncTableKV } from "@cocalc/nats/sync/synctable-kv"; +import { createSyncTable } from "@cocalc/nats/sync/synctable"; import { sha1 } from "@cocalc/backend/misc_node"; const logger = getLogger("database:nats"); @@ -72,11 +72,12 @@ async function getResponse({ name, args, account_id, project_id, nc }) { async function createChangefeed(opts, nc) { const query = opts.query; const env = { nc, jc, sha1 }; - const synctable = new SyncTableKV({ + const synctable = createSyncTable({ query, env, account_id: opts.account_id, project_id: opts.project_id, + atomic: true, }); await synctable.init(); const f = (cb) => { @@ -99,8 +100,16 @@ async function createChangefeed(opts, nc) { const { action, new_val, old_val } = result as any; // action = 'insert', 'update', 'delete', 'close' // e.g., {"action":"insert","new_val":{"title":"testingxxxxx","project_id":"81e0c408-ac65-4114-bad5-5f4b6539bd0e"}} - if (action == "insert" || action == "update") { + if (action == "insert") { await synctable.set(new_val); + } else if (action == "update") { + // update -- since atomic have to get the current value; + // this of course assumes there is one process writing to + // this part of the key value store (the atomic business). + await synctable.set({ + ...(await synctable.get(new_val)), + ...new_val, + }); } else if (action == "delete") { await synctable.delete(old_val); } diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index d134d5fcf0..f7225332ad 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -10,6 +10,7 @@ import { parse_query } from "@cocalc/sync/table/util"; import sha1 from "sha1"; import { keys } from "lodash"; import { type HubApi, initHubApi } from "@cocalc/nats/api/index"; +import { uuid } from "@cocalc/util/misc"; export class NatsClient { /*private*/ client: WebappClient; @@ -118,8 +119,12 @@ export class NatsClient { return await js.consumers.get(stream); }; - synctable = async (query, obj?): Promise => { + synctable = async ( + query, + options?: { obj?: object; atomic?: boolean; stream?: boolean }, + ): Promise => { query = parse_query(query); + const obj = options?.obj; if (obj != null) { const table = keys(query)[0]; for (const k in obj) { @@ -127,6 +132,7 @@ export class NatsClient { } } const s = createSyncTable({ + ...options, query, env: { sha1, @@ -138,4 +144,29 @@ export class NatsClient { await s.init(); return s; }; + + changefeedInterest = async (query, noError?: boolean) => { + // express interest + const changes = uuid(); + // (re-)start changefeed going + try { + await this.client.nats_client.callHub({ + service: "db", + name: "userQuery", + args: [{ changes, query }], + }); + } catch (err) { + if (noError) { + console.warn(err); + return; + } else { + throw err; + } + } + }; + + changefeed = async (query) => { + this.changefeedInterest(query, true); + return await this.synctable(query, { atomic: true }); + }; } diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts new file mode 100644 index 0000000000..d2669c9b71 --- /dev/null +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -0,0 +1,105 @@ +import sha1 from "sha1"; +import { keys } from "lodash"; +import { client_db } from "@cocalc/util/db-schema/client-db"; +import { getKv, toKey, type NatsEnv, natsKeyPrefix } from "./synctable-kv"; + +export class SyncTableKVAtomic { + private kv?; + private nc; + private jc; + private sha1; + public readonly natsKeyPrefix; + public readonly table; + private primaryKeys: string[]; + private project_id?: string; + private account_id?: string; + + constructor({ + query, + env, + account_id, + project_id, + }: { + query; + env: NatsEnv; + account_id?: string; + project_id?: string; + }) { + this.sha1 = env.sha1 ?? sha1; + this.nc = env.nc; + this.jc = env.jc; + const table = keys(query)[0]; + this.table = table; + this.natsKeyPrefix = natsKeyPrefix({ query, atomic: true }); + this.project_id = project_id ?? query[table][0].project_id; + this.account_id = account_id ?? query[table][0].account_id; + this.primaryKeys = client_db.primary_keys(table); + } + + init = async () => { + this.kv = await getKv({ + nc: this.nc, + project_id: this.project_id, + account_id: this.account_id, + }); + }; + + private primaryString = (obj): string => { + if (this.primaryKeys.length === 1) { + return toKey(obj[this.primaryKeys[0]] ?? "")!; + } else { + // compound primary key + return toKey(this.primaryKeys.map((pk) => obj[pk]))!; + } + }; + + private natObjectKey = (obj): string => { + if (obj == null) { + throw Error("obj must be an object (not null)"); + } + return this.sha1(this.primaryString(obj)); + }; + + private getKey = (obj): string => { + return `${this.natsKeyPrefix}.${this.natObjectKey(obj)}`; + }; + + set = async (obj) => { + const key = this.getKey(obj); + const value = this.jc.encode(obj); + await this.kv.put(key, value); + }; + + delete = async (obj) => { + await this.kv.delete(this.getKey(obj)); + }; + + private decode = (mesg) => { + return mesg?.sm?.data != null ? this.jc.decode(mesg.sm.data) : null; + }; + + get = async (obj?) => { + if (obj == null) { + // everything + const keys = await this.kv.keys(`${this.natsKeyPrefix}.>`); + const all: any = {}; + for await (const key of keys) { + const value = this.decode(await this.kv.get(key)); + all[this.primaryString(value)] = value; + } + return all; + } + return this.decode(await this.kv.get(this.getKey(obj))); + }; + + // watch for changes + async *watch() { + const w = await this.kv.watch({ + key: `${this.natsKeyPrefix}.>`, + }); + for await (const { value } of w) { + const obj = this.jc.decode(value); + yield { [this.primaryString(obj)]: obj }; + } + } +} diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 80b9ed1b59..373b0592f9 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -14,6 +14,16 @@ import jsonStableStringify from "json-stable-stringify"; import { keys } from "lodash"; import { client_db } from "@cocalc/util/db-schema/client-db"; +export function natsKeyPrefix({ + query, + atomic = false, +}: { + query; + atomic?: boolean; +}) { + return sha1(jsonStableStringify({ query, atomic })); +} + export async function getKv({ nc, project_id, @@ -38,14 +48,14 @@ export async function getKv({ return await kvm.create(name, { compression: true }); } -interface NatsEnv { +export interface NatsEnv { nc; // nats connection jc; // jsoncodec // compute sha1 hash efficiently (set differently on backend) sha1?: (string) => string; } -function toKey(x): string | undefined { +export function toKey(x): string | undefined { if (x === undefined) { return undefined; } else if (typeof x === "object") { @@ -61,6 +71,7 @@ export class SyncTableKV { private jc; private sha1; public readonly table; + public readonly natsKeyPrefix; private primaryKeys: string[]; private primaryKeysSet: Set; private fields: string[]; @@ -83,6 +94,7 @@ export class SyncTableKV { this.jc = env.jc; const table = keys(query)[0]; this.table = table; + this.natsKeyPrefix = natsKeyPrefix({ query, atomic: false }); this.project_id = project_id ?? query[table][0].project_id; this.account_id = account_id ?? query[table][0].account_id; this.primaryKeys = client_db.primary_keys(table); @@ -117,7 +129,7 @@ export class SyncTableKV { }; private getKey = (obj, field?: string): string => { - const x = `${this.table}.${this.natObjectKey(obj)}`; + const x = `${this.natsKeyPrefix}.${this.natObjectKey(obj)}`; if (field == null) { return x; } else { @@ -143,15 +155,8 @@ export class SyncTableKV { get = async (obj?, field?) => { if (obj == null) { - // everything known in this table by the project - // TODO: this is way to broad -- we probably need to restructure things - // by adding in a hash of the query? e.g., {this.table}.{hash(query)}.(same as before), - // since there's many queries for one table and below gives too much and we can't client - // side do all the filters...? or shouldn't (too slow). But also worry about clutter - // and maybe ttl? - - const keys = await this.kv.keys(`${this.table}.>`); + const keys = await this.kv.keys(`${this.natsKeyPrefix}.>`); const all: any = {}; for await (const key of keys) { const mesg = await this.kv.get(key); diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts index 29a259533b..6381610404 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/nats/sync/synctable-stream.ts @@ -40,7 +40,7 @@ export class SyncTableStream extends EventEmitter { private nc; private jc; private sha1; - private table; + public readonly table; private primaryKeys: string[]; private project_id?: string; private account_id?: string; @@ -246,6 +246,10 @@ export class SyncTableStream extends EventEmitter { this.set_state("closed"); }; + delete = async (_obj) => { + throw Error("delete: not implemented for stream synctable"); + }; + // no-op because we always immediately publish changes on set. save = () => {}; has_uncommitted_changes = () => { diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/nats/sync/synctable.ts index c8d90f8a73..29d830ceed 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/nats/sync/synctable.ts @@ -1,23 +1,33 @@ -import { SyncTableKV } from "./synctable-kv"; +import { SyncTableKV, type NatsEnv } from "./synctable-kv"; +import { SyncTableKVAtomic } from "./synctable-kv-atomic"; import { SyncTableStream } from "./synctable-stream"; -import { keys } from "lodash"; -export type SyncTable = SyncTableKV | SyncTableStream; +export type SyncTable = SyncTableKV | SyncTableStream | SyncTableKVAtomic; export function createSyncTable({ query, env, account_id, project_id, + atomic, + stream, }: { query; - env; - account_id?; - project_id?; + env: NatsEnv; + account_id?: string; + project_id?: string; + atomic?: boolean; + stream?: boolean; }) { - const table = keys(query)[0]; - if (table == "patches") { + if (stream) { + if (atomic) { + throw Error("atomic stream not implemented yet"); + } return new SyncTableStream({ query, env, account_id, project_id }); } - return new SyncTableKV({ query, env, account_id, project_id }); + if (atomic) { + return new SyncTableKVAtomic({ query, env, account_id, project_id }); + } else { + return new SyncTableKV({ query, env, account_id, project_id }); + } } diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index 946c9a0e67..8bddb815d1 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -501,8 +501,8 @@ export class Client extends EventEmitter implements ProjectClientInterface { return the_synctable; } - synctable_nats = async (query, obj?) => { - return await synctable_nats(query, obj); + synctable_nats = async (query, options?) => { + return await synctable_nats(query, options); }; // WARNING: making two of the exact same sync_string or sync_db will definitely diff --git a/src/packages/project/nats/synctable.ts b/src/packages/project/nats/synctable.ts index 6fd701fd41..1d77d8feb6 100644 --- a/src/packages/project/nats/synctable.ts +++ b/src/packages/project/nats/synctable.ts @@ -10,23 +10,26 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; const jc = JSONCodec(); const cache: { [key: string]: SyncTable } = {}; -const synctable = reuseInFlight(async (query, obj?) => { - const key = JSON.stringify(query); - if (cache[key] == null) { - const nc = await getConnection(); - query = parse_query(query); - const table = keys(query)[0]; - if (obj != null) { - for (const k in obj) { - query[table][0][k] = obj[k]; +const synctable = reuseInFlight( + async (query, options: { obj?; atomic?: boolean; stream?: boolean }) => { + const key = JSON.stringify(query); + if (cache[key] == null) { + const nc = await getConnection(); + query = parse_query(query); + const table = keys(query)[0]; + const obj = options?.obj; + if (obj != null) { + for (const k in obj) { + query[table][0][k] = obj[k]; + } } + query[table][0].project_id = project_id; + const s = createSyncTable({ ...options, query, env: { sha1, jc, nc } }); + await s.init(); + cache[key] = s; } - query[table][0].project_id = project_id; - const s = createSyncTable({ query, env: { sha1, jc, nc } }); - await s.init(); - cache[key] = s; - } - return cache[key]; -}); + return cache[key]; + }, +); export default synctable; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index b0a6648f19..d6ab375a46 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1253,6 +1253,7 @@ export class SyncDoc extends EventEmitter { synctable = await this.client.synctable_nats(query, { project_id: this.project_id, path: this.path, + stream: true, }); } else { switch (this.data_server) { From a6a04e4142df0d28c9aecffd39e91c00ce7ef3c5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 26 Jan 2025 18:46:36 +0000 Subject: [PATCH 044/281] nats database changefeeds: one per query --- src/packages/database/nats/index.ts | 117 +++++++++++++++++----------- 1 file changed, 70 insertions(+), 47 deletions(-) diff --git a/src/packages/database/nats/index.ts b/src/packages/database/nats/index.ts index 604f767d04..f2d2d4e644 100644 --- a/src/packages/database/nats/index.ts +++ b/src/packages/database/nats/index.ts @@ -14,6 +14,8 @@ import { callback } from "awaiting"; import { db } from "@cocalc/database"; import { createSyncTable } from "@cocalc/nats/sync/synctable"; import { sha1 } from "@cocalc/backend/misc_node"; +import jsonStableStringify from "json-stable-stringify"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; const logger = getLogger("database:nats"); @@ -69,52 +71,73 @@ async function getResponse({ name, args, account_id, project_id, nc }) { // This is tricky. We return the first result as a normal // async function, but then handle (and don't return) // the subsequent calls to cb generated by the changefeed. -async function createChangefeed(opts, nc) { - const query = opts.query; - const env = { nc, jc, sha1 }; - const synctable = createSyncTable({ - query, - env, - account_id: opts.account_id, - project_id: opts.project_id, - atomic: true, - }); - await synctable.init(); - const f = (cb) => { - let first = true; - db().user_query({ - ...opts, - cb: async (err, result) => { - if (first) { - first = false; - cb(err, result); - if (result != null) { - for (const x of result[synctable.table]) { - logger.debug("changefeed init", x); - await synctable.set(x); +const changefeedInterest: { [hash: string]: number } = {}; + +const createChangefeed = reuseInFlight( + async (opts, nc) => { + const query = opts.query; + const hash = sha1(jsonStableStringify(query)); + const now = Date.now(); + if (changefeedInterest[hash]) { + changefeedInterest[hash] = now; + logger.debug("using existing changefeed for", query); + return; + } + logger.debug("creating new changefeed for", query); + const env = { nc, jc, sha1 }; + const synctable = createSyncTable({ + query, + env, + account_id: opts.account_id, + project_id: opts.project_id, + atomic: true, + }); + await synctable.init(); + const f = (cb) => { + let first = true; + db().user_query({ + ...opts, + cb: async (err, result) => { + if (first) { + first = false; + cb(err); + if (result != null) { + for (const x of result[synctable.table]) { + logger.debug("changefeed init", x); + await synctable.set(x); + } } + return; } - return; - } - logger.debug("changefeed", result); - const { action, new_val, old_val } = result as any; - // action = 'insert', 'update', 'delete', 'close' - // e.g., {"action":"insert","new_val":{"title":"testingxxxxx","project_id":"81e0c408-ac65-4114-bad5-5f4b6539bd0e"}} - if (action == "insert") { - await synctable.set(new_val); - } else if (action == "update") { - // update -- since atomic have to get the current value; - // this of course assumes there is one process writing to - // this part of the key value store (the atomic business). - await synctable.set({ - ...(await synctable.get(new_val)), - ...new_val, - }); - } else if (action == "delete") { - await synctable.delete(old_val); - } - }, - }); - }; - return await callback(f); -} + logger.debug("changefeed", result); + const { action, new_val, old_val } = result as any; + // action = 'insert', 'update', 'delete', 'close' + // e.g., {"action":"insert","new_val":{"title":"testingxxxxx","project_id":"81e0c408-ac65-4114-bad5-5f4b6539bd0e"}} + if (action == "insert") { + await synctable.set(new_val); + } else if (action == "update") { + // update -- since atomic have to get the current value; + // this of course assumes there is one process writing to + // this part of the key value store (the atomic business). + await synctable.set({ + ...(await synctable.get(new_val)), + ...new_val, + }); + } else if (action == "delete") { + await synctable.delete(old_val); + } else if (action == "close") { + delete changefeedInterest[hash]; + } + }, + }); + }; + try { + await callback(f); + // it's running successfully + changefeedInterest[hash] = Date.now(); + } catch (err) { + throw err; + } + }, + { createKey: (args) => jsonStableStringify(args[0]) }, +); From 4e4fda0170c7fb43ed8f15996caf7bb12bb4c29f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 26 Jan 2025 19:07:02 +0000 Subject: [PATCH 045/281] nats db synctable: automatically stop --- docs/nats/devlog.md | 15 +++++++++++++ src/packages/database/nats/index.ts | 33 ++++++++++++++++++++++++++-- src/packages/frontend/client/nats.ts | 4 +--- src/packages/nats/sync/synctable.ts | 4 ++++ 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 6030381fe4..2a7d1572be 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -273,6 +273,21 @@ With the above algorithm, it should be very easy to reimplement the client side - If you refresh your browser, everything stays stable \-\- nothing changes at all and you instantly have your data. Same if the network drops and resumes. - When implementing our new synctable, we can immediately start with the possibly stale data from the last time it was active, then update it to the correct data. Thus even if everything but NATS is done/unavailable, the experience would be much better. It's like "local first", but somehow "network mesh first". With a leaf node it would literally be local first. +--- + +This is working well! + +TODO: + +- [ ] build full SyncTable on top of my current implementation of synctablekvatomic, to make sure it is sufficient + +THEN do the following to make it robust and scalable + +- [ ] store in nats which servers are actively managing which synctables +- [ ] store in nats the client interest data, instead of storing it in memory in a server? i.e., instead of client making an api call, they could instead just update a kv and say "i am interested in this changefeed". This approach would make everything just keep working easily even as servers scale up/down/restart. + +--- + ## [ ] Goal: Terminal and **compute server** Another thing to do for compute servers: diff --git a/src/packages/database/nats/index.ts b/src/packages/database/nats/index.ts index f2d2d4e644..e32de47ac0 100644 --- a/src/packages/database/nats/index.ts +++ b/src/packages/database/nats/index.ts @@ -12,10 +12,15 @@ import { getConnection } from "@cocalc/backend/nats"; import { getUserId } from "@cocalc/nats/api"; import { callback } from "awaiting"; import { db } from "@cocalc/database"; -import { createSyncTable } from "@cocalc/nats/sync/synctable"; +import { + createSyncTable, + CHANGEFEED_INTEREST_PERIOD_MS, +} from "@cocalc/nats/sync/synctable"; import { sha1 } from "@cocalc/backend/misc_node"; import jsonStableStringify from "json-stable-stringify"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { uuid } from "@cocalc/util/misc"; +import { delay } from "awaiting"; const logger = getLogger("database:nats"); @@ -58,7 +63,8 @@ async function handleRequest(mesg, nc) { async function getResponse({ name, args, account_id, project_id, nc }) { if (name == "userQuery") { const opts = { ...args[0], account_id, project_id }; - if (opts.changes == null) { + if (!opts.changes) { + // a normal query return await userQuery(opts); } else { return await createChangefeed(opts, nc); @@ -84,6 +90,7 @@ const createChangefeed = reuseInFlight( return; } logger.debug("creating new changefeed for", query); + const changes = uuid(); const env = { nc, jc, sha1 }; const synctable = createSyncTable({ query, @@ -97,6 +104,7 @@ const createChangefeed = reuseInFlight( let first = true; db().user_query({ ...opts, + changes, cb: async (err, result) => { if (first) { first = false; @@ -135,6 +143,27 @@ const createChangefeed = reuseInFlight( await callback(f); // it's running successfully changefeedInterest[hash] = Date.now(); + + const watch = async () => { + // it's all setup and running. If there's no interest for a while, stop watching + while (true) { + await delay(CHANGEFEED_INTEREST_PERIOD_MS); + if ( + Date.now() - changefeedInterest[hash] > + CHANGEFEED_INTEREST_PERIOD_MS + ) { + logger.debug( + "insufficient interest in the changefeed, so we stop it.", + query, + ); + db().user_query_cancel_changefeed({ id: changes }); + delete changefeedInterest[hash]; + } + } + }; + // do not block on this. + watch(); + return; } catch (err) { throw err; } diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index f7225332ad..eff33f5781 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -10,7 +10,6 @@ import { parse_query } from "@cocalc/sync/table/util"; import sha1 from "sha1"; import { keys } from "lodash"; import { type HubApi, initHubApi } from "@cocalc/nats/api/index"; -import { uuid } from "@cocalc/util/misc"; export class NatsClient { /*private*/ client: WebappClient; @@ -147,13 +146,12 @@ export class NatsClient { changefeedInterest = async (query, noError?: boolean) => { // express interest - const changes = uuid(); // (re-)start changefeed going try { await this.client.nats_client.callHub({ service: "db", name: "userQuery", - args: [{ changes, query }], + args: [{ changes:true, query }], }); } catch (err) { if (noError) { diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/nats/sync/synctable.ts index 29d830ceed..997591d6f6 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/nats/sync/synctable.ts @@ -4,6 +4,10 @@ import { SyncTableStream } from "./synctable-stream"; export type SyncTable = SyncTableKV | SyncTableStream | SyncTableKVAtomic; +// When the database is watching tables for changefeeds, if it doesn't get a clear expression +// of interest from a client every this much time, it automatically stops. +export const CHANGEFEED_INTEREST_PERIOD_MS = 120000; + export function createSyncTable({ query, env, From f7fbc450d36dcace3db0f30c201032137659e258 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 26 Jan 2025 19:31:12 +0000 Subject: [PATCH 046/281] organize sync changefeed code --- docs/nats/devlog.md | 2 +- src/packages/sync/table/changefeed.ts | 55 +++++++++++++-------------- src/packages/sync/table/synctable.ts | 8 ++-- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 2a7d1572be..42dbdf6436 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -279,7 +279,7 @@ This is working well! TODO: -- [ ] build full SyncTable on top of my current implementation of synctablekvatomic, to make sure it is sufficient +- [ ] build full SyncTable on top of my current implementation of synctablekvatomic, to _make sure it is sufficient_ THEN do the following to make it robust and scalable diff --git a/src/packages/sync/table/changefeed.ts b/src/packages/sync/table/changefeed.ts index 3174a96c51..ce725e93c4 100644 --- a/src/packages/sync/table/changefeed.ts +++ b/src/packages/sync/table/changefeed.ts @@ -44,10 +44,10 @@ export class Changefeed extends EventEmitter { // changefeed, and return the initial state // of the table. Throws an exception if anything // goes wrong. - public async connect(): Promise { + connect = async () => { if (this.state != "disconnected") { throw Error( - `can only connect if state is 'disconnected' but it is ${this.state}` + `can only connect if state is 'disconnected' but it is ${this.state}`, ); } this.state = "connecting"; @@ -66,11 +66,27 @@ export class Changefeed extends EventEmitter { this.state = "connected"; this.process_queue_next_tick(); return resp.query[this.table]; - } + }; + + close = (): void => { + this.state = "closed"; + if (this.id != null) { + // stop listening for future updates + this.cancel_query(this.id); + } + this.emit("close"); + this.removeAllListeners(); + close(this); + this.state = "closed"; + }; + + get_state = (): string => { + return this.state; + }; // Wait a tick, then process the queue of messages that // arrived during initialization. - private async process_queue_next_tick(): Promise { + private process_queue_next_tick = async () => { await delay(0); while (this.state != "closed" && this.handle_update_queue.length > 0) { const x = this.handle_update_queue.shift(); @@ -78,9 +94,9 @@ export class Changefeed extends EventEmitter { this.handle_update(x.err, x.resp); } } - } + }; - private run_the_query(cb: Function): void { + private run_the_query = (cb: Function): void => { // This query_function gets called first on the // initial query, then repeatedly with each changefeed // update. The input function "cb" will be called @@ -102,9 +118,9 @@ export class Changefeed extends EventEmitter { } }, }); - } + }; - private handle_update(err, resp): void { + private handle_update = (err, resp): void => { if (this.state != "connected") { if (this.state == "closed") { // expected, since last updates after query cancel may get through... @@ -139,21 +155,8 @@ export class Changefeed extends EventEmitter { } x.action = resp.action; this.emit("update", x); - } - - public close(): void { - this.state = "closed"; - if (this.id != null) { - // stop listening for future updates - this.cancel_query(this.id); - } - this.emit("close"); - this.removeAllListeners(); - close(this); - this.state = "closed"; - } - - private async cancel_query(id: string): Promise { + }; + private cancel_query = async (id: string) => { try { await this.query_cancel(id); } catch (err) { @@ -161,11 +164,7 @@ export class Changefeed extends EventEmitter { // Basically anything that could cause an error would have also // canceled the changefeed anyways. } - } - - public get_state(): string { - return this.state; - } + }; } // diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index 8a6be53e4c..bf7b4f21ff 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -32,6 +32,10 @@ import { query_function } from "./query-function"; import { assert_uuid, copy, is_array, is_object, len } from "@cocalc/util/misc"; import * as schema from "@cocalc/util/schema"; import mergeDeep from "@cocalc/util/immutable-deep-merge"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { Changefeed } from "./changefeed"; +import { parse_query, to_key } from "./util"; + import type { Client } from "@cocalc/sync/client/types"; export type { Client }; @@ -54,10 +58,6 @@ function is_fatal(err: string): boolean { return err.indexOf("FATAL") != -1; } -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; - -import { Changefeed } from "./changefeed"; -import { parse_query, to_key } from "./util"; export type State = "disconnected" | "connected" | "closed"; From cee658cf49eaf19eb45b5863eb02028748ea57b9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 26 Jan 2025 20:18:33 +0000 Subject: [PATCH 047/281] nats synctables: got page to load and the non-project-specific tables to actually work :-) --- src/packages/frontend/client/nats.ts | 2 +- src/packages/nats/sync/synctable-kv-atomic.ts | 3 +- src/packages/nats/sync/synctable-kv.ts | 3 - src/packages/sync/table/changefeed-nats.ts | 60 +++++++++++++++++++ src/packages/sync/table/changefeed.ts | 2 +- src/packages/sync/table/synctable.ts | 14 ++++- 6 files changed, 74 insertions(+), 10 deletions(-) create mode 100644 src/packages/sync/table/changefeed-nats.ts diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index eff33f5781..b3ac99e6e3 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -151,7 +151,7 @@ export class NatsClient { await this.client.nats_client.callHub({ service: "db", name: "userQuery", - args: [{ changes:true, query }], + args: [{ changes: true, query }], }); } catch (err) { if (noError) { diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index d2669c9b71..dd0e0eb4d7 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -98,8 +98,7 @@ export class SyncTableKVAtomic { key: `${this.natsKeyPrefix}.>`, }); for await (const { value } of w) { - const obj = this.jc.decode(value); - yield { [this.primaryString(obj)]: obj }; + yield this.jc.decode(value); } } } diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 373b0592f9..38958ca976 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -33,9 +33,6 @@ export async function getKv({ project_id?: string; account_id?: string; }) { - if (account_id && project_id) { - throw Error("both account_id and project_id can't be defined"); - } let name; if (account_id) { name = `account-${account_id}`; diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts new file mode 100644 index 0000000000..b5e4352abc --- /dev/null +++ b/src/packages/sync/table/changefeed-nats.ts @@ -0,0 +1,60 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +import { EventEmitter } from "events"; +import type { State } from "./changefeed"; +import { delay } from "awaiting"; + +export class NatsChangefeed extends EventEmitter { + private client; + private query; + private options; + private state: State = "disconnected"; + private natsSynctable?; + + constructor({ client, query, options }: { client; query; options }) { + super(); + this.client = client; + this.query = query; + this.options = options; + console.log('changefeed-nats', this.query, this.options); + } + + connect = async () => { + this.natsSynctable = await this.client.nats_client.changefeed(this.query); + this.interest(); + this.watch(); + return Object.values(await this.natsSynctable.get()); + }; + + close = (): void => { + this.state = "closed"; + this.emit("close"); + }; + + get_state = (): string => { + return this.state; + }; + + private interest = async () => { + await delay(30000); + while (this.state != "closed") { + // console.log("express interest in", this.query); + await this.client.nats_client.changefeedInterest(this.query); + await delay(30000); + } + }; + private watch = async () => { + if (this.natsSynctable == null) { + return; + } + for await (const new_val of await this.natsSynctable.watch()) { + if (this.state == "closed") { + return; + } + this.emit("update", { action: "update", new_val }); + } + }; +} diff --git a/src/packages/sync/table/changefeed.ts b/src/packages/sync/table/changefeed.ts index ce725e93c4..4b3b1470ed 100644 --- a/src/packages/sync/table/changefeed.ts +++ b/src/packages/sync/table/changefeed.ts @@ -7,7 +7,7 @@ import { EventEmitter } from "events"; import { callback, delay } from "awaiting"; import { close } from "@cocalc/util/misc"; -type State = "closed" | "disconnected" | "connecting" | "connected"; +export type State = "closed" | "disconnected" | "connecting" | "connected"; export class Changefeed extends EventEmitter { private query: any; diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index bf7b4f21ff..c2d922fb64 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -34,6 +34,7 @@ import * as schema from "@cocalc/util/schema"; import mergeDeep from "@cocalc/util/immutable-deep-merge"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { Changefeed } from "./changefeed"; +import { NatsChangefeed } from "./changefeed-nats"; import { parse_query, to_key } from "./util"; import type { Client } from "@cocalc/sync/client/types"; @@ -58,11 +59,10 @@ function is_fatal(err: string): boolean { return err.indexOf("FATAL") != -1; } - export type State = "disconnected" | "connected" | "closed"; export class SyncTable extends EventEmitter { - private changefeed?: Changefeed; + private changefeed?: Changefeed | NatsChangefeed; private query: Query; private client_query: any; private primary_keys: string[]; @@ -729,7 +729,15 @@ export class SyncTable extends EventEmitter { let delay_ms: number = 500; while (true) { this.close_changefeed(); - this.changefeed = new Changefeed(this.changefeed_options()); + if (this.client.is_browser()) { + this.changefeed = new NatsChangefeed({ + client: this.client, + query: this.query, + options: this.options, + }); + } else { + this.changefeed = new Changefeed(this.changefeed_options()); + } await this.wait_until_ready_to_query_db(); try { return await this.changefeed.connect(); From 854ec97f728d1ad8edebd197a2b527a84224161b Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 26 Jan 2025 21:28:20 +0000 Subject: [PATCH 048/281] nats: fix collab editing --- src/packages/sync/client/synctable-project.ts | 2 +- src/packages/sync/editor/generic/sync-doc.ts | 12 +++++++----- src/packages/sync/table/synctable.ts | 5 +++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/packages/sync/client/synctable-project.ts b/src/packages/sync/client/synctable-project.ts index 4f4b54fe3c..4dc651bc43 100644 --- a/src/packages/sync/client/synctable-project.ts +++ b/src/packages/sync/client/synctable-project.ts @@ -9,7 +9,7 @@ Synctable that uses the project websocket rather than the database. import { delay } from "awaiting"; -import { SyncTable, synctable_no_database } from "@cocalc/sync/table"; +import { type SyncTable, synctable_no_database } from "@cocalc/sync/table"; import { once, retry_until_success } from "@cocalc/util/async-utils"; import { assertDefined } from "@cocalc/util/misc"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index d6ab375a46..8344e20254 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1251,8 +1251,10 @@ export class SyncDoc extends EventEmitter { let synctable; if (this.useNats && query.patches) { synctable = await this.client.synctable_nats(query, { - project_id: this.project_id, - path: this.path, + obj: { + project_id: this.project_id, + path: this.path, + }, stream: true, }); } else { @@ -1341,9 +1343,9 @@ export class SyncDoc extends EventEmitter { // Used for internal debug logging private dbg = (f: string = ""): Function => { - if (this.useNats) { - return (...args) => console.log(f, ...args); - } + // if (this.useNats) { + // return (...args) => console.log(f, ...args); + // } return this.client?.dbg(`SyncDoc('${this.path}').${f}`); }; diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index c2d922fb64..f9af6f2807 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -130,7 +130,8 @@ export class SyncTable extends EventEmitter { // entirely by the project (e.g., sync-doc support). private no_db_set: boolean = false; - // Set only for some tables. + // Set only for some tables that are hosted directly on a project (not database), + // e.g., the project_status and listings. private project_id?: string; private last_has_uncommitted_changes?: boolean = undefined; @@ -729,7 +730,7 @@ export class SyncTable extends EventEmitter { let delay_ms: number = 500; while (true) { this.close_changefeed(); - if (this.client.is_browser()) { + if (this.client.is_browser() && !this.project_id) { this.changefeed = new NatsChangefeed({ client: this.client, query: this.query, From 9a69f84cc19cfa971cfd3b470b182fb91964d625 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 27 Jan 2025 01:45:20 +0000 Subject: [PATCH 049/281] clean up use of sha1 (js version all imported from @cocalc/util); also implement sha1base64, which may be of use someday; more unit tests --- src/packages/backend/sha1.test.ts | 27 ++- src/packages/backend/sha1.ts | 15 +- src/packages/frontend/client/console.ts | 1 - .../frame-editors/x11-editor/xpra-client.ts | 4 +- src/packages/frontend/package.json | 3 +- src/packages/frontend/search/embeddings.ts | 4 +- .../next/lib/share/proxy/get-public-path.ts | 4 +- src/packages/next/package.json | 1 - src/packages/pnpm-lock.yaml | 217 ++++++++---------- .../sync/editor/generic/ipywidgets-state.ts | 2 +- src/packages/sync/package.json | 3 +- src/packages/util/db-schema/client-db.ts | 4 +- src/packages/util/misc.ts | 13 ++ 13 files changed, 151 insertions(+), 147 deletions(-) diff --git a/src/packages/backend/sha1.test.ts b/src/packages/backend/sha1.test.ts index c4e680ad25..ab087c32f8 100644 --- a/src/packages/backend/sha1.test.ts +++ b/src/packages/backend/sha1.test.ts @@ -1,4 +1,5 @@ -import { sha1, uuidsha1 } from "./sha1"; +import { sha1, sha1base64, uuidsha1 } from "./sha1"; +import * as misc from "@cocalc/util/misc"; const cocalc = "CoCalc"; const hash = "c898c97dca68742a5a6331f9fa0ca02483cbfd25"; @@ -30,3 +31,27 @@ describe("UUIDs", () => { expect(uuidsha1(Buffer.from(cocalc))).toBe(uuid); }); }); + +describe("compare this nodejs implementation to the pure javascript implementation", () => { + it("CoCalc/string", () => { + expect(sha1(cocalc)).toBe(misc.sha1(cocalc)); + }); + it("hash", () => { + expect(sha1(hash)).toBe(misc.sha1(hash)); + }); + it("uuid", () => { + expect(sha1(uuid)).toBe(misc.sha1(uuid)); + }); +}); + +describe("base64: compare this nodejs implementation to the pure javascript implementation", () => { + it("CoCalc/string", () => { + expect(sha1base64(cocalc)).toBe(misc.sha1base64(cocalc)); + }); + it("hash", () => { + expect(sha1base64(hash)).toBe(misc.sha1base64(hash)); + }); + it("uuid", () => { + expect(sha1base64(uuid)).toBe(misc.sha1base64(uuid)); + }); +}); diff --git a/src/packages/backend/sha1.ts b/src/packages/backend/sha1.ts index c3863c1c78..bf96f2aff1 100644 --- a/src/packages/backend/sha1.ts +++ b/src/packages/backend/sha1.ts @@ -2,12 +2,14 @@ sha1 hash functionality */ -import { createHash } from "crypto"; +import { createHash, type BinaryToTextEncoding } from "crypto"; // compute sha1 hash of data in hex -export function sha1(data: Buffer | string): string { +export function sha1( + data: Buffer | string, + encoding: BinaryToTextEncoding = "hex", +): string { const sha1sum = createHash("sha1"); - if (typeof data === "string") { sha1sum.update(data, "utf8"); } else { @@ -20,11 +22,16 @@ export function sha1(data: Buffer | string): string { sha1sum.update(uint8Array); } - return sha1sum.digest("hex"); + return sha1sum.digest(encoding); +} + +export function sha1base64(data: Buffer | string): string { + return sha1(data, "base64"); } // Compute a uuid v4 from the Sha-1 hash of data. // Optionally, if knownSha1 is given, just uses that, rather than recomputing it. +// WARNING: try to avoid using this, since it discards information! export function uuidsha1(data: Buffer | string, knownSha1?: string): string { const s = knownSha1 ?? sha1(data); let i = -1; diff --git a/src/packages/frontend/client/console.ts b/src/packages/frontend/client/console.ts index 40b251445c..cce4f183ff 100644 --- a/src/packages/frontend/client/console.ts +++ b/src/packages/frontend/client/console.ts @@ -48,7 +48,6 @@ export function setup_global_cocalc(client): void { cocalc.misc = require("@cocalc/util/misc"); cocalc.immutable = require("immutable"); cocalc.done = cocalc.misc.done; - cocalc.sha1 = require("sha1"); cocalc.prom_client = require("../prom-client"); cocalc.schema = require("@cocalc/util/schema"); cocalc.redux = redux; diff --git a/src/packages/frontend/frame-editors/x11-editor/xpra-client.ts b/src/packages/frontend/frame-editors/x11-editor/xpra-client.ts index ac3b81cbac..474e7ca39e 100644 --- a/src/packages/frontend/frame-editors/x11-editor/xpra-client.ts +++ b/src/packages/frontend/frame-editors/x11-editor/xpra-client.ts @@ -7,12 +7,11 @@ import { join } from "path"; import { throttle } from "underscore"; - import { alert_message } from "@cocalc/frontend/alerts"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import { open_new_tab } from "@cocalc/frontend/misc"; import { retry_until_success } from "@cocalc/util/async-utils"; -import { close, hash_string } from "@cocalc/util/misc"; +import { close, hash_string, sha1 } from "@cocalc/util/misc"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { ConnectionStatus } from "../frame-tree/types"; import { ExecOutput, touch, touch_project } from "../generic/client"; @@ -20,7 +19,6 @@ import { ExecOpts0, XpraServer } from "./xpra-server"; import { Client } from "./xpra/client"; import { Surface } from "./xpra/surface"; import { is_copy } from "./xpra/util"; -const sha1 = require("sha1"); const BASE_DPI: number = 96; diff --git a/src/packages/frontend/package.json b/src/packages/frontend/package.json index 8ea6303873..6a8ab110fe 100644 --- a/src/packages/frontend/package.json +++ b/src/packages/frontend/package.json @@ -43,10 +43,10 @@ "@cocalc/comm": "workspace:*", "@cocalc/frontend": "workspace:*", "@cocalc/jupyter": "workspace:*", + "@cocalc/local-storage-lru": "^2.4.3", "@cocalc/nats": "workspace:*", "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", - "@cocalc/local-storage-lru": "^2.4.3", "@cocalc/widgets": "^1.2.0", "@dnd-kit/core": "^6.0.7", "@dnd-kit/modifiers": "^6.0.1", @@ -157,7 +157,6 @@ "react-redux": "^8.0.5", "react-timeago": "^7.2.0", "react-virtuoso": "^4.9.0", - "sha1": "^1.1.1", "shallowequal": "^1.1.0", "shell-escape": "^0.2.0", "slate": "^0.103.0", diff --git a/src/packages/frontend/search/embeddings.ts b/src/packages/frontend/search/embeddings.ts index 34e7a473e6..d421b805b0 100644 --- a/src/packages/frontend/search/embeddings.ts +++ b/src/packages/frontend/search/embeddings.ts @@ -25,13 +25,11 @@ are not "fatal data loss" for us, since this is just search. import jsonStable from "json-stable-stringify"; import { debounce } from "lodash"; -import sha1 from "sha1"; - import { webapp_client } from "@cocalc/frontend/webapp-client"; import type { SyncDB } from "@cocalc/sync/editor/db"; import type { Document } from "@cocalc/sync/editor/generic/types"; import { EmbeddingData } from "@cocalc/util/db-schema/llm"; -import { close, copy_with, len, uuidsha1 } from "@cocalc/util/misc"; +import { close, copy_with, len, sha1, uuidsha1 } from "@cocalc/util/misc"; // How long until we update the index, if users stops using this file actively. const DEBOUNCE_MS = 7500; diff --git a/src/packages/next/lib/share/proxy/get-public-path.ts b/src/packages/next/lib/share/proxy/get-public-path.ts index 31c90c640a..d747642333 100644 --- a/src/packages/next/lib/share/proxy/get-public-path.ts +++ b/src/packages/next/lib/share/proxy/get-public-path.ts @@ -17,7 +17,7 @@ what is publicly shared (at some unit), and it's nice if it is useful (e.g., for import getProxyProjectId from "lib/share/proxy/project"; import getPool from "@cocalc/database/pool"; -import * as sha1 from "sha1"; +import { sha1 } from "@cocalc/util/misc"; import { fileInGist } from "./api"; export function shouldUseProxy(owner: string): boolean { @@ -116,7 +116,7 @@ export default async function getProxyPublicPath({ const now = new Date(); await pool.query( "INSERT INTO public_paths (id, url, project_id, path, description, last_edited, last_saved, created) VALUES($1, $2, $3, $4, $5, $6, $7, $8)", - [id, publicPathUrl, project_id, path, description, now, now, now] + [id, publicPathUrl, project_id, path, description, now, now, now], ); return { id, diff --git a/src/packages/next/package.json b/src/packages/next/package.json index acd7e4f20a..b13561d40c 100644 --- a/src/packages/next/package.json +++ b/src/packages/next/package.json @@ -91,7 +91,6 @@ "react-google-recaptcha-v3": "^1.9.7", "react-intl": "^7.1.0", "serve-index": "^1.9.1", - "sha1": "^1.1.1", "sharp": "^0.32.6", "timeago-react": "^3.0.4", "tslib": "^2.3.1", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 2748115531..33e128d64d 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -610,9 +610,6 @@ importers: react-virtuoso: specifier: ^4.9.0 version: 4.10.4(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - sha1: - specifier: ^1.1.1 - version: 1.1.1 shallowequal: specifier: ^1.1.0 version: 1.1.0 @@ -1199,9 +1196,6 @@ importers: serve-index: specifier: ^1.9.1 version: 1.9.1 - sha1: - specifier: ^1.1.1 - version: 1.1.1 sharp: specifier: ^0.32.6 version: 0.32.6 @@ -1904,7 +1898,7 @@ importers: version: 3.0.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0 events: specifier: 3.3.0 version: 3.3.0 @@ -1917,9 +1911,6 @@ importers: lodash: specifier: ^4.17.21 version: 4.17.21 - sha1: - specifier: ^1.1.1 - version: 1.1.1 devDependencies: '@types/node': specifier: ^18.16.14 @@ -2271,10 +2262,6 @@ packages: resolution: {integrity: sha512-0xZJFNE5XMpENsgfHYTw8FbX4kv53mFLn2i3XPoq69LyhYSCBJtitaHx9QnsVTrsogI4Z3+HtEfZ2/GFPOtf5g==} engines: {node: '>=6.9.0'} - '@babel/code-frame@7.25.9': - resolution: {integrity: sha512-z88xeGxnzehn2sqZ8UdGQEvYErF1odv2CftxInpSYJt6uHuPe9YjahKZITGs3l5LeI9d2ROG+obuDAoSlqbNfQ==} - engines: {node: '>=6.9.0'} - '@babel/code-frame@7.26.2': resolution: {integrity: sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==} engines: {node: '>=6.9.0'} @@ -2453,10 +2440,6 @@ packages: resolution: {integrity: sha512-iYyACpW3iW8Fw+ZybQK+drQre+ns/tKpXbNESfrhNnPLIklLbXr7MYJ6gPEd0iETGLOK+SxMjVvKb/ffmk+FEw==} engines: {node: '>=6.9.0'} - '@babel/highlight@7.25.9': - resolution: {integrity: sha512-llL88JShoCsth8fF8R4SJnIn+WLvR6ccFxu1H3FlMhDontdcmZWf2HgIZ7AIqV3Xcck1idlohrN4EUBQz6klbw==} - engines: {node: '>=6.9.0'} - '@babel/parser@7.25.6': resolution: {integrity: sha512-trGdfBdbD0l1ZPmcJ83eNxB9rbEax4ALFTF7fN386TMYbeCQbyme5cOEXQhbGXKebwGaB/J52w1mrklMcbgy6Q==} engines: {node: '>=6.0.0'} @@ -2467,11 +2450,6 @@ packages: engines: {node: '>=6.0.0'} hasBin: true - '@babel/parser@7.25.9': - resolution: {integrity: sha512-aI3jjAAO1fh7vY/pBGsn1i9LDbRP43+asrRlkPuTXW5yHXtd1NgTEMudbBoDDxrf1daEEfPJqR+JBMakzrR4Dg==} - engines: {node: '>=6.0.0'} - hasBin: true - '@babel/parser@7.26.5': resolution: {integrity: sha512-SRJ4jYmXRqV1/Xc+TIVG84WjHBXKlxO9sHQnA2Pf12QQEAp1LOh6kDzNHXcUnbH1QI0FDoPPVOt+vyUDucxpaw==} engines: {node: '>=6.0.0'} @@ -2612,10 +2590,6 @@ packages: resolution: {integrity: sha512-JWtuCu8VQsMladxVz/P4HzHUGCAwpuqacmowgXFs5XjxIgKuNjnLokQzuVjlTvIzODaDmpjT3oxcC48vyk9EWg==} engines: {node: '>=6.9.0'} - '@babel/types@7.25.9': - resolution: {integrity: sha512-OwS2CM5KocvQ/k7dFJa8i5bNGJP0hXWfVCfDkqRFP1IreH1JDC7wG6eCYCi0+McbfT8OR/kNqsI0UU0xP9H6PQ==} - engines: {node: '>=6.9.0'} - '@babel/types@7.26.5': resolution: {integrity: sha512-L6mZmwFDK6Cjh1nRCLXpa6no13ZIioJDz7mdkzHv399pThrTa/k0nUlNaenOeh2kWu/iaOQYElEpKPUswUa9Vg==} engines: {node: '>=6.9.0'} @@ -12870,7 +12844,7 @@ snapshots: '@ampproject/remapping@2.3.0': dependencies: - '@jridgewell/gen-mapping': 0.3.5 + '@jridgewell/gen-mapping': 0.3.8 '@jridgewell/trace-mapping': 0.3.25 '@ant-design/colors@6.0.0': @@ -12994,26 +12968,21 @@ snapshots: dependencies: '@babel/highlight': 7.25.7 picocolors: 1.1.1 - - '@babel/code-frame@7.25.9': - dependencies: - '@babel/highlight': 7.25.9 - picocolors: 1.1.1 + optional: true '@babel/code-frame@7.26.2': dependencies: '@babel/helper-validator-identifier': 7.25.9 js-tokens: 4.0.0 picocolors: 1.1.1 - optional: true '@babel/compat-data@7.25.4': {} - '@babel/compat-data@7.25.8': {} - - '@babel/compat-data@7.26.5': + '@babel/compat-data@7.25.8': optional: true + '@babel/compat-data@7.26.5': {} + '@babel/core@7.25.2': dependencies: '@ampproject/remapping': 2.3.0 @@ -13027,7 +12996,7 @@ snapshots: '@babel/traverse': 7.25.6 '@babel/types': 7.25.6 convert-source-map: 2.0.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -13073,6 +13042,7 @@ snapshots: semver: 6.3.1 transitivePeerDependencies: - supports-color + optional: true '@babel/core@7.26.0': dependencies: @@ -13087,13 +13057,12 @@ snapshots: '@babel/traverse': 7.26.5 '@babel/types': 7.26.5 convert-source-map: 2.0.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 transitivePeerDependencies: - supports-color - optional: true '@babel/generator@7.25.6': dependencies: @@ -13108,6 +13077,7 @@ snapshots: '@jridgewell/gen-mapping': 0.3.5 '@jridgewell/trace-mapping': 0.3.25 jsesc: 3.0.2 + optional: true '@babel/generator@7.26.5': dependencies: @@ -13116,7 +13086,6 @@ snapshots: '@jridgewell/gen-mapping': 0.3.8 '@jridgewell/trace-mapping': 0.3.25 jsesc: 3.1.0 - optional: true '@babel/helper-annotate-as-pure@7.24.7': dependencies: @@ -13137,6 +13106,7 @@ snapshots: browserslist: 4.24.0 lru-cache: 5.1.1 semver: 6.3.1 + optional: true '@babel/helper-compilation-targets@7.26.5': dependencies: @@ -13145,7 +13115,6 @@ snapshots: browserslist: 4.24.4 lru-cache: 5.1.1 semver: 6.3.1 - optional: true '@babel/helper-create-class-features-plugin@7.25.4(@babel/core@7.25.2)': dependencies: @@ -13187,6 +13156,7 @@ snapshots: '@babel/types': 7.25.8 transitivePeerDependencies: - supports-color + optional: true '@babel/helper-module-imports@7.25.9': dependencies: @@ -13194,7 +13164,6 @@ snapshots: '@babel/types': 7.26.5 transitivePeerDependencies: - supports-color - optional: true '@babel/helper-module-transforms@7.25.2(@babel/core@7.25.2(supports-color@9.4.0))(supports-color@9.4.0)': dependencies: @@ -13225,6 +13194,7 @@ snapshots: '@babel/traverse': 7.25.7 transitivePeerDependencies: - supports-color + optional: true '@babel/helper-module-transforms@7.26.0(@babel/core@7.26.0)': dependencies: @@ -13234,7 +13204,6 @@ snapshots: '@babel/traverse': 7.26.5 transitivePeerDependencies: - supports-color - optional: true '@babel/helper-optimise-call-expression@7.24.7': dependencies: @@ -13271,6 +13240,7 @@ snapshots: '@babel/types': 7.25.8 transitivePeerDependencies: - supports-color + optional: true '@babel/helper-skip-transparent-expression-wrappers@7.24.7': dependencies: @@ -13281,23 +13251,25 @@ snapshots: '@babel/helper-string-parser@7.24.8': {} - '@babel/helper-string-parser@7.25.7': {} + '@babel/helper-string-parser@7.25.7': + optional: true '@babel/helper-string-parser@7.25.9': {} '@babel/helper-validator-identifier@7.24.7': {} - '@babel/helper-validator-identifier@7.25.7': {} + '@babel/helper-validator-identifier@7.25.7': + optional: true '@babel/helper-validator-identifier@7.25.9': {} '@babel/helper-validator-option@7.24.8': {} - '@babel/helper-validator-option@7.25.7': {} - - '@babel/helper-validator-option@7.25.9': + '@babel/helper-validator-option@7.25.7': optional: true + '@babel/helper-validator-option@7.25.9': {} + '@babel/helpers@7.25.6': dependencies: '@babel/template': 7.25.0 @@ -13307,12 +13279,12 @@ snapshots: dependencies: '@babel/template': 7.25.9 '@babel/types': 7.25.8 + optional: true '@babel/helpers@7.26.0': dependencies: '@babel/template': 7.25.9 '@babel/types': 7.26.5 - optional: true '@babel/highlight@7.24.7': dependencies: @@ -13327,13 +13299,7 @@ snapshots: chalk: 2.4.2 js-tokens: 4.0.0 picocolors: 1.1.1 - - '@babel/highlight@7.25.9': - dependencies: - '@babel/helper-validator-identifier': 7.25.9 - chalk: 2.4.2 - js-tokens: 4.0.0 - picocolors: 1.1.1 + optional: true '@babel/parser@7.25.6': dependencies: @@ -13342,166 +13308,162 @@ snapshots: '@babel/parser@7.25.8': dependencies: '@babel/types': 7.25.8 - - '@babel/parser@7.25.9': - dependencies: - '@babel/types': 7.25.9 + optional: true '@babel/parser@7.26.5': dependencies: '@babel/types': 7.26.5 - optional: true '@babel/plugin-syntax-async-generators@7.8.4(@babel/core@7.25.8)': dependencies: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 + optional: true '@babel/plugin-syntax-async-generators@7.8.4(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 - optional: true '@babel/plugin-syntax-bigint@7.8.3(@babel/core@7.25.8)': dependencies: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 + optional: true '@babel/plugin-syntax-bigint@7.8.3(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 - optional: true '@babel/plugin-syntax-class-properties@7.12.13(@babel/core@7.25.8)': dependencies: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 + optional: true '@babel/plugin-syntax-class-properties@7.12.13(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 - optional: true '@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.25.8)': dependencies: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 + optional: true '@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 - optional: true '@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.25.8)': dependencies: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 + optional: true '@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 - optional: true '@babel/plugin-syntax-jsx@7.24.7(@babel/core@7.25.2)': dependencies: '@babel/core': 7.25.2 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-jsx@7.24.7(@babel/core@7.25.8)': + '@babel/plugin-syntax-jsx@7.24.7(@babel/core@7.26.0)': dependencies: - '@babel/core': 7.25.8 + '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 '@babel/plugin-syntax-logical-assignment-operators@7.10.4(@babel/core@7.25.8)': dependencies: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 + optional: true '@babel/plugin-syntax-logical-assignment-operators@7.10.4(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 - optional: true '@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.25.8)': dependencies: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 + optional: true '@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 - optional: true '@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.25.8)': dependencies: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 + optional: true '@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 - optional: true '@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.25.8)': dependencies: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 + optional: true '@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 - optional: true '@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.25.8)': dependencies: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 + optional: true '@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 - optional: true '@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.25.8)': dependencies: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 + optional: true '@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 - optional: true '@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.25.8)': dependencies: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 + optional: true '@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 - optional: true '@babel/plugin-syntax-typescript@7.25.4(@babel/core@7.25.2)': dependencies: '@babel/core': 7.25.2 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-typescript@7.25.4(@babel/core@7.25.8)': + '@babel/plugin-syntax-typescript@7.25.4(@babel/core@7.26.0)': dependencies: - '@babel/core': 7.25.8 + '@babel/core': 7.26.0 '@babel/helper-plugin-utils': 7.24.8 '@babel/plugin-transform-modules-commonjs@7.24.8(@babel/core@7.25.2)': @@ -13559,9 +13521,9 @@ snapshots: '@babel/template@7.25.9': dependencies: - '@babel/code-frame': 7.25.9 - '@babel/parser': 7.25.9 - '@babel/types': 7.25.9 + '@babel/code-frame': 7.26.2 + '@babel/parser': 7.26.5 + '@babel/types': 7.26.5 '@babel/traverse@7.25.6': dependencies: @@ -13570,7 +13532,7 @@ snapshots: '@babel/parser': 7.25.6 '@babel/template': 7.25.0 '@babel/types': 7.25.6 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0 globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -13598,6 +13560,7 @@ snapshots: globals: 11.12.0 transitivePeerDependencies: - supports-color + optional: true '@babel/traverse@7.26.5': dependencies: @@ -13606,11 +13569,10 @@ snapshots: '@babel/parser': 7.26.5 '@babel/template': 7.25.9 '@babel/types': 7.26.5 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0 globals: 11.12.0 transitivePeerDependencies: - supports-color - optional: true '@babel/types@7.25.6': dependencies: @@ -13623,17 +13585,12 @@ snapshots: '@babel/helper-string-parser': 7.25.7 '@babel/helper-validator-identifier': 7.25.7 to-fast-properties: 2.0.0 - - '@babel/types@7.25.9': - dependencies: - '@babel/helper-string-parser': 7.25.9 - '@babel/helper-validator-identifier': 7.25.9 + optional: true '@babel/types@7.26.5': dependencies: '@babel/helper-string-parser': 7.25.9 '@babel/helper-validator-identifier': 7.25.9 - optional: true '@bcoe/v8-coverage@0.2.3': {} @@ -14411,7 +14368,7 @@ snapshots: '@jest/transform@29.7.0': dependencies: - '@babel/core': 7.25.8 + '@babel/core': 7.26.0 '@jest/types': 29.6.3 '@jridgewell/trace-mapping': 0.3.25 babel-plugin-istanbul: 6.1.1 @@ -14942,7 +14899,7 @@ snapshots: '@nestjs/axios@3.0.3(@nestjs/common@10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1))(axios@1.7.4)(rxjs@7.8.1)': dependencies: '@nestjs/common': 10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1) - axios: 1.7.4(debug@4.4.0) + axios: 1.7.4 rxjs: 7.8.1 '@nestjs/common@10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1)': @@ -15086,7 +15043,7 @@ snapshots: '@nestjs/common': 10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1) '@nestjs/core': 10.4.3(@nestjs/common@10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1))(encoding@0.1.13)(reflect-metadata@0.1.13)(rxjs@7.8.1) '@nuxtjs/opencollective': 0.3.2(encoding@0.1.13) - axios: 1.7.4(debug@4.4.0) + axios: 1.7.4 chalk: 4.1.2 commander: 8.3.0 compare-versions: 4.1.4 @@ -15648,24 +15605,24 @@ snapshots: '@types/babel__core@7.20.5': dependencies: - '@babel/parser': 7.25.8 - '@babel/types': 7.25.8 + '@babel/parser': 7.26.5 + '@babel/types': 7.26.5 '@types/babel__generator': 7.6.8 '@types/babel__template': 7.4.4 '@types/babel__traverse': 7.20.5 '@types/babel__generator@7.6.8': dependencies: - '@babel/types': 7.25.8 + '@babel/types': 7.26.5 '@types/babel__template@7.4.4': dependencies: - '@babel/parser': 7.25.8 - '@babel/types': 7.25.8 + '@babel/parser': 7.26.5 + '@babel/types': 7.26.5 '@types/babel__traverse@7.20.5': dependencies: - '@babel/types': 7.25.8 + '@babel/types': 7.26.5 '@types/backbone@1.4.14': dependencies: @@ -16354,7 +16311,7 @@ snapshots: agent-base@7.1.1: dependencies: - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0 transitivePeerDependencies: - supports-color @@ -16661,6 +16618,14 @@ snapshots: awaiting@3.0.0: {} + axios@1.7.4: + dependencies: + follow-redirects: 1.15.9 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + axios@1.7.4(debug@4.4.0): dependencies: follow-redirects: 1.15.9(debug@4.4.0) @@ -16691,6 +16656,7 @@ snapshots: slash: 3.0.0 transitivePeerDependencies: - supports-color + optional: true babel-jest@29.7.0(@babel/core@7.26.0): dependencies: @@ -16704,7 +16670,6 @@ snapshots: slash: 3.0.0 transitivePeerDependencies: - supports-color - optional: true babel-loader@9.2.1(@babel/core@7.25.2)(webpack@5.97.1): dependencies: @@ -16726,7 +16691,7 @@ snapshots: babel-plugin-jest-hoist@29.6.3: dependencies: '@babel/template': 7.25.9 - '@babel/types': 7.25.8 + '@babel/types': 7.26.5 '@types/babel__core': 7.20.5 '@types/babel__traverse': 7.20.5 @@ -16749,6 +16714,7 @@ snapshots: '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.25.8) '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.25.8) '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.25.8) + optional: true babel-preset-current-node-syntax@1.0.1(@babel/core@7.26.0): dependencies: @@ -16765,20 +16731,19 @@ snapshots: '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.26.0) '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.26.0) '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.26.0) - optional: true babel-preset-jest@29.6.3(@babel/core@7.25.8): dependencies: '@babel/core': 7.25.8 babel-plugin-jest-hoist: 29.6.3 babel-preset-current-node-syntax: 1.0.1(@babel/core@7.25.8) + optional: true babel-preset-jest@29.6.3(@babel/core@7.26.0): dependencies: '@babel/core': 7.26.0 babel-plugin-jest-hoist: 29.6.3 babel-preset-current-node-syntax: 1.0.1(@babel/core@7.26.0) - optional: true babel-runtime@6.26.0: dependencies: @@ -19097,6 +19062,8 @@ snapshots: optionalDependencies: debug: 4.4.0(supports-color@8.1.1) + follow-redirects@1.15.9: {} + follow-redirects@1.15.9(debug@4.4.0): optionalDependencies: debug: 4.4.0(supports-color@8.1.1) @@ -19944,7 +19911,7 @@ snapshots: https-proxy-agent@7.0.4: dependencies: agent-base: 7.1.1 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0 transitivePeerDependencies: - supports-color @@ -20365,8 +20332,8 @@ snapshots: istanbul-lib-instrument@5.2.1: dependencies: - '@babel/core': 7.25.8 - '@babel/parser': 7.25.8 + '@babel/core': 7.26.0 + '@babel/parser': 7.26.5 '@istanbuljs/schema': 0.1.3 istanbul-lib-coverage: 3.2.2 semver: 6.3.1 @@ -20375,8 +20342,8 @@ snapshots: istanbul-lib-instrument@6.0.3: dependencies: - '@babel/core': 7.25.8 - '@babel/parser': 7.25.8 + '@babel/core': 7.26.0 + '@babel/parser': 7.26.5 '@istanbuljs/schema': 0.1.3 istanbul-lib-coverage: 3.2.2 semver: 7.6.3 @@ -20400,7 +20367,7 @@ snapshots: istanbul-lib-source-maps@4.0.1: dependencies: - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0 istanbul-lib-coverage: 3.2.2 source-map: 0.6.1 transitivePeerDependencies: @@ -20516,10 +20483,10 @@ snapshots: jest-config@29.7.0(@types/node@18.19.50): dependencies: - '@babel/core': 7.25.8 + '@babel/core': 7.26.0 '@jest/test-sequencer': 29.7.0 '@jest/types': 29.6.3 - babel-jest: 29.7.0(@babel/core@7.25.8) + babel-jest: 29.7.0(@babel/core@7.26.0) chalk: 4.1.2 ci-info: 3.9.0 deepmerge: 4.3.1 @@ -20546,10 +20513,10 @@ snapshots: jest-config@29.7.0(@types/node@18.19.71): dependencies: - '@babel/core': 7.25.8 + '@babel/core': 7.26.0 '@jest/test-sequencer': 29.7.0 '@jest/types': 29.6.3 - babel-jest: 29.7.0(@babel/core@7.25.8) + babel-jest: 29.7.0(@babel/core@7.26.0) chalk: 4.1.2 ci-info: 3.9.0 deepmerge: 4.3.1 @@ -20662,7 +20629,7 @@ snapshots: jest-message-util@29.7.0: dependencies: - '@babel/code-frame': 7.25.7 + '@babel/code-frame': 7.26.2 '@jest/types': 29.6.3 '@types/stack-utils': 2.0.3 chalk: 4.1.2 @@ -20760,15 +20727,15 @@ snapshots: jest-snapshot@29.7.0: dependencies: - '@babel/core': 7.25.8 - '@babel/generator': 7.25.7 - '@babel/plugin-syntax-jsx': 7.24.7(@babel/core@7.25.8) - '@babel/plugin-syntax-typescript': 7.25.4(@babel/core@7.25.8) - '@babel/types': 7.25.8 + '@babel/core': 7.26.0 + '@babel/generator': 7.26.5 + '@babel/plugin-syntax-jsx': 7.24.7(@babel/core@7.26.0) + '@babel/plugin-syntax-typescript': 7.25.4(@babel/core@7.26.0) + '@babel/types': 7.26.5 '@jest/expect-utils': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - babel-preset-current-node-syntax: 1.0.1(@babel/core@7.25.8) + babel-preset-current-node-syntax: 1.0.1(@babel/core@7.26.0) chalk: 4.1.2 expect: 29.7.0 graceful-fs: 4.2.11 @@ -20921,11 +20888,11 @@ snapshots: jsesc@2.5.2: {} - jsesc@3.0.2: {} - - jsesc@3.1.0: + jsesc@3.0.2: optional: true + jsesc@3.1.0: {} + json-bigint@1.0.0: dependencies: bignumber.js: 9.1.2 @@ -22249,7 +22216,7 @@ snapshots: parse-json@5.2.0: dependencies: - '@babel/code-frame': 7.24.7 + '@babel/code-frame': 7.26.2 error-ex: 1.3.2 json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 diff --git a/src/packages/sync/editor/generic/ipywidgets-state.ts b/src/packages/sync/editor/generic/ipywidgets-state.ts index 3db5c132b3..ed08e01e8a 100644 --- a/src/packages/sync/editor/generic/ipywidgets-state.ts +++ b/src/packages/sync/editor/generic/ipywidgets-state.ts @@ -17,12 +17,12 @@ import { is_object, len, auxFileToOriginal, + sha1, } from "@cocalc/util/misc"; import { SyncDoc } from "./sync-doc"; import { SyncTable } from "@cocalc/sync/table/synctable"; import { Client } from "./types"; import { debounce } from "lodash"; -import sha1 from "sha1"; type State = "init" | "ready" | "closed"; diff --git a/src/packages/sync/package.json b/src/packages/sync/package.json index 130fcdfa60..2907733a2c 100644 --- a/src/packages/sync/package.json +++ b/src/packages/sync/package.json @@ -38,8 +38,7 @@ "events": "3.3.0", "immutable": "^4.3.0", "json-stable-stringify": "^1.0.1", - "lodash": "^4.17.21", - "sha1": "^1.1.1" + "lodash": "^4.17.21" }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/sync", "repository": { diff --git a/src/packages/util/db-schema/client-db.ts b/src/packages/util/db-schema/client-db.ts index 1354dea8ff..d26f0f1bcd 100644 --- a/src/packages/util/db-schema/client-db.ts +++ b/src/packages/util/db-schema/client-db.ts @@ -7,8 +7,8 @@ import { is_array } from "../misc"; import { SCHEMA } from "./index"; +import { sha1 } from "@cocalc/util/misc"; -const sha1 = require("sha1"); class ClientDB { private _primary_keys_cache; public r; @@ -69,7 +69,7 @@ class ClientDB { const v = t != null ? t.primary_key : undefined; if (v == null) { throw Error( - `primary key for table '${table}' must be explicitly specified in schema` + `primary key for table '${table}' must be explicitly specified in schema`, ); } if (typeof v === "string") { diff --git a/src/packages/util/misc.ts b/src/packages/util/misc.ts index 08b0ce1494..98318c7f1d 100644 --- a/src/packages/util/misc.ts +++ b/src/packages/util/misc.ts @@ -69,6 +69,19 @@ export { import sha1 from "sha1"; export { sha1 }; +function base16ToBase64(hex) { + return Buffer.from(hex, 'hex').toString('base64') +// let bytes: number[] = []; +// for (let c = 0; c < hex.length; c += 2) { +// bytes.push(parseInt(hex.substr(c, 2), 16)); +// } +// return btoa(String.fromCharCode.apply(null, bytes)); +} + +export function sha1base64(s) { + return base16ToBase64(sha1(s)); +} + import getRandomValues from "get-random-values"; import * as lodash from "lodash"; import * as immutable from "immutable"; From b33ec2471285f185dd97b34e7eede94bfed71526 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 27 Jan 2025 01:47:51 +0000 Subject: [PATCH 050/281] more sha1 cleanup --- src/packages/frontend/client/nats.ts | 2 +- .../terminal-editor/nats-terminal-connection.ts | 3 +-- src/packages/nats/README.md | 4 ++++ src/packages/nats/sync/synctable-kv-atomic.ts | 2 +- src/packages/nats/sync/synctable-kv.ts | 2 +- src/packages/nats/sync/synctable-stream.ts | 3 +-- src/packages/project/nats/synctable.ts | 6 +++++- 7 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 src/packages/nats/README.md diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index b3ac99e6e3..bdfe42cfe1 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -7,7 +7,7 @@ import { redux } from "../app-framework"; import * as jetstream from "@nats-io/jetstream"; import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; import { parse_query } from "@cocalc/sync/table/util"; -import sha1 from "sha1"; +import { sha1 } from "@cocalc/util/misc"; import { keys } from "lodash"; import { type HubApi, initHubApi } from "@cocalc/nats/api/index"; diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index c382e698f9..db89ee8e68 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -1,9 +1,8 @@ import { webapp_client } from "@cocalc/frontend/webapp-client"; import { EventEmitter } from "events"; import { JSONCodec } from "nats.ws"; -import sha1 from "sha1"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { uuid } from "@cocalc/util/misc"; +import { sha1, uuid } from "@cocalc/util/misc"; import { delay } from "awaiting"; const jc = JSONCodec(); diff --git a/src/packages/nats/README.md b/src/packages/nats/README.md new file mode 100644 index 0000000000..889426f35d --- /dev/null +++ b/src/packages/nats/README.md @@ -0,0 +1,4 @@ +### NOTES + +- we are using sha1 hashes a lot because we have to store various data, e.g., arbitrary file paths, as nats segments, so MUST have a bounded size string with simple characters. + The very low probability of a collision is discussed here: https://crypto.stackexchange.com/questions/2583/is-it-fair-to-assume-that-sha1-collisions-wont-occur-on-a-set-of-100k-strings diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index dd0e0eb4d7..20aca0ff4c 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -1,7 +1,7 @@ -import sha1 from "sha1"; import { keys } from "lodash"; import { client_db } from "@cocalc/util/db-schema/client-db"; import { getKv, toKey, type NatsEnv, natsKeyPrefix } from "./synctable-kv"; +import { sha1 } from "@cocalc/util/misc"; export class SyncTableKVAtomic { private kv?; diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 38958ca976..30492775e2 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -9,7 +9,7 @@ It uses a SINGLE NATS key-value store to represent */ import { Kvm } from "@nats-io/kv"; -import sha1 from "sha1"; +import { sha1 } from "@cocalc/util/misc"; import jsonStableStringify from "json-stable-stringify"; import { keys } from "lodash"; import { client_db } from "@cocalc/util/db-schema/client-db"; diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts index 6381610404..58d7beb68a 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/nats/sync/synctable-stream.ts @@ -10,10 +10,9 @@ It uses a NATS stream to store the elements in a well defined order. */ import { jetstreamManager, jetstream } from "@nats-io/jetstream"; -import sha1 from "sha1"; import jsonStableStringify from "json-stable-stringify"; import { keys } from "lodash"; -import { cmp_Date, is_array, isValidUUID } from "@cocalc/util/misc"; +import { cmp_Date, is_array, isValidUUID, sha1 } from "@cocalc/util/misc"; import { client_db } from "@cocalc/util/db-schema/client-db"; import { EventEmitter } from "events"; diff --git a/src/packages/project/nats/synctable.ts b/src/packages/project/nats/synctable.ts index 1d77d8feb6..339be82efc 100644 --- a/src/packages/project/nats/synctable.ts +++ b/src/packages/project/nats/synctable.ts @@ -24,7 +24,11 @@ const synctable = reuseInFlight( } } query[table][0].project_id = project_id; - const s = createSyncTable({ ...options, query, env: { sha1, jc, nc } }); + const s = createSyncTable({ + ...options, + query, + env: { sha1, jc, nc }, + }); await s.init(); cache[key] = s; } From 7d8b198e0fb38b816077cbc8a908e761a323f6f9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 27 Jan 2025 04:57:38 +0000 Subject: [PATCH 051/281] nats api: refactoring --- docs/nats/devlog.md | 3 +- src/packages/nats/api/index.ts | 71 +++++++++++++++++++++++++++++---- src/packages/nats/tsconfig.json | 2 +- src/packages/server/nats/api.ts | 29 ++++++++------ 4 files changed, 83 insertions(+), 22 deletions(-) diff --git a/docs/nats/devlog.md b/docs/nats/devlog.md index 42dbdf6436..f1d89fe55e 100644 --- a/docs/nats/devlog.md +++ b/docs/nats/devlog.md @@ -279,7 +279,8 @@ This is working well! TODO: -- [ ] build full SyncTable on top of my current implementation of synctablekvatomic, to _make sure it is sufficient_ +- [x] build full proof of concept SyncTable on top of my current implementation of synctablekvatomic, to _make sure it is sufficient_ + - this worked and wasn't too difficult THEN do the following to make it robust and scalable diff --git a/src/packages/nats/api/index.ts b/src/packages/nats/api/index.ts index 263ba095bf..b4a9028b6c 100644 --- a/src/packages/nats/api/index.ts +++ b/src/packages/nats/api/index.ts @@ -2,18 +2,73 @@ import type { Customize } from "@cocalc/util/db-schema/server-settings"; import { isValidUUID } from "@cocalc/util/misc"; export interface HubApi { - getCustomize: (fields?: string[]) => Promise; - userQuery: (opts: { - project_id?: string; - query: any; - options?: any[]; - }) => Promise; + system: { + getCustomize: (fields?: string[]) => Promise; + }; + + db: { + userQuery: (opts: { + project_id?: string; + account_id?: string; + query: any; + options?: any[]; + }) => Promise; + }; + + purchases: { + getBalance: ({ account_id }) => Promise; + getMinBalance: (account_id) => Promise; + }; +} + +const authFirst = ({ args, account_id, project_id }) => { + if (args[0] == null) { + args[0] = {} as any; + } + if (account_id) { + args[0].account_id = account_id; + } else if (project_id) { + args[0].project_id = project_id; + } + return args; +}; + +const noAuth = ({ args }) => args; + +const HubApiStructure = { + system: { + getCustomize: noAuth, + }, + db: { + userQuery: authFirst, + }, + purchases: { + getBalance: ({ account_id }) => { + return [{ account_id }]; + }, + getMinBalance: ({ account_id }) => [account_id], + }, +} as const; + +export function transformArgs({ name, args, account_id, project_id }) { + const [group, functionName] = name.split("."); + return HubApiStructure[group]?.[functionName]({ + args, + account_id, + project_id, + }); } export function initHubApi(callHubApi): HubApi { const hubApi: any = {}; - for (const name of ["getCustomize", "userQuery"]) { - hubApi[name] = async (...args) => await callHubApi({ name, args }); + for (const group in HubApiStructure) { + if (hubApi[group] == null) { + hubApi[group] = {}; + } + for (const functionName in HubApiStructure[group]) { + hubApi[group][functionName] = async (...args) => + await callHubApi({ name: `${group}.${functionName}`, args }); + } } return hubApi as HubApi; } diff --git a/src/packages/nats/tsconfig.json b/src/packages/nats/tsconfig.json index 61be42b5d0..6cdb913e19 100644 --- a/src/packages/nats/tsconfig.json +++ b/src/packages/nats/tsconfig.json @@ -5,5 +5,5 @@ "outDir": "dist" }, "exclude": ["node_modules", "dist", "test"], - "references": [{ "path": "../database" }] + "references": [{ "path": "../util" }] } diff --git a/src/packages/server/nats/api.ts b/src/packages/server/nats/api.ts index 8cdac3d658..b5b025d161 100644 --- a/src/packages/server/nats/api.ts +++ b/src/packages/server/nats/api.ts @@ -26,7 +26,7 @@ To view all requests (and replies) in realtime: import { JSONCodec } from "nats"; import getLogger from "@cocalc/backend/logger"; -import { type HubApi, getUserId } from "@cocalc/nats/api/index"; +import { type HubApi, getUserId, transformArgs } from "@cocalc/nats/api/index"; const logger = getLogger("server:nats:api"); @@ -65,23 +65,28 @@ async function handleApiRequest(mesg) { import userQuery from "@cocalc/database/user-query"; import getCustomize from "@cocalc/database/settings/customize"; +import getBalance from "@cocalc/server/purchases/get-balance"; +import getMinBalance from "@cocalc/server/purchases/get-min-balance"; const hubApi: HubApi = { - getCustomize, - userQuery, + system: { + getCustomize, + }, + db: { + userQuery, + }, + purchases: { + getBalance, + getMinBalance, + }, }; async function getResponse({ name, args, account_id, project_id }) { - const f = hubApi[name]; + const [group, functionName] = name.split("."); + const f = hubApi[group]?.[functionName]; if (f == null) { throw Error(`unknown function '${name}'`); } - if (name == "userQuery" && args[0] != null) { - if (account_id) { - args[0].account_id = account_id; - } else if (project_id) { - args[0].project_id = project_id; - } - } - return await f(...args); + const args2 = transformArgs({ name, args, account_id, project_id }); + return await f(...args2); } From 4ed5f3f7e5f882bf03ca7080759dfc78b8e8a549 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 27 Jan 2025 06:09:54 +0000 Subject: [PATCH 052/281] nats: refactoring hub api -- rewrite time functionality to use NATS instead of old coffeescript websocket --- src/packages/frontend/client/hub.ts | 2 +- src/packages/frontend/client/time.ts | 66 ++++++++----------- src/packages/hub/client.coffee | 3 - src/packages/nats/api/db.ts | 14 ++++ src/packages/nats/api/index.ts | 52 +++------------ src/packages/nats/api/purchases.ts | 11 ++++ src/packages/nats/api/system.ts | 12 ++++ src/packages/nats/api/util.ts | 13 ++++ src/packages/server/nats/api/db.ts | 3 + .../server/nats/{api.ts => api/index.ts} | 27 +++----- src/packages/server/nats/api/purchases.ts | 4 ++ src/packages/server/nats/api/system.ts | 6 ++ src/packages/server/nats/index.ts | 5 +- src/packages/server/package.json | 1 + 14 files changed, 113 insertions(+), 106 deletions(-) create mode 100644 src/packages/nats/api/db.ts create mode 100644 src/packages/nats/api/purchases.ts create mode 100644 src/packages/nats/api/system.ts create mode 100644 src/packages/nats/api/util.ts create mode 100644 src/packages/server/nats/api/db.ts rename src/packages/server/nats/{api.ts => api/index.ts} (83%) create mode 100644 src/packages/server/nats/api/purchases.ts create mode 100644 src/packages/server/nats/api/system.ts diff --git a/src/packages/frontend/client/hub.ts b/src/packages/frontend/client/hub.ts index c0487b2625..a4692eb223 100644 --- a/src/packages/frontend/client/hub.ts +++ b/src/packages/frontend/client/hub.ts @@ -113,7 +113,7 @@ export class HubClient { } public send(mesg: object): void { - //console.log("send at #{misc.mswalltime()}", mesg) + // console.log("send to hub", mesg); const data = to_json_socket(mesg); this.mesg_data.sent_length += data.length; this.emit_mesg_data(); diff --git a/src/packages/frontend/client/time.ts b/src/packages/frontend/client/time.ts index 2b094c53af..76e2fb4060 100644 --- a/src/packages/frontend/client/time.ts +++ b/src/packages/frontend/client/time.ts @@ -4,20 +4,18 @@ */ import { delay } from "awaiting"; - import { get_local_storage, set_local_storage, } from "@cocalc/frontend/misc/local-storage"; -import * as message from "@cocalc/util/message"; export class TimeClient { private client: any; private ping_interval_ms: number = 30000; // interval in ms between pings - private last_ping: Date = new Date(0); - private last_pong?: { server: Date; local: Date }; + private last_ping: number = 0; + private last_pong?: { server: number; local: number }; private clock_skew_ms?: number; - private last_server_time?: Date; + private last_server_time?: number; private closed: boolean = false; constructor(client: any) { @@ -29,31 +27,26 @@ export class TimeClient { } // Ping server and also use the ping to determine clock skew. - public async ping(noLoop: boolean = false): Promise { - if (this.closed) return; - const start = (this.last_ping = new Date()); + ping = async (noLoop: boolean = false): Promise => { + if (this.closed) { + return; + } + const start = (this.last_ping = Date.now()); let pong; try { - pong = await this.client.async_call({ - allow_post: false, - message: message.ping(), - timeout: 10, // CRITICAL that this timeout be less than the @_ping_interval - }); + pong = await this.client.nats_client.hub.system.ping(); } catch (err) { if (!noLoop) { // try again **sooner** - setTimeout(this.ping.bind(this), this.ping_interval_ms / 2); + setTimeout(this.ping, this.ping_interval_ms / 2); } return; } - const now = new Date(); + const now = Date.now(); // Only record something if success, got a pong, and the round trip is short! // If user messes with their clock during a ping and we don't do this, then // bad things will happen. - if ( - pong?.event == "pong" && - now.valueOf() - this.last_ping.valueOf() <= 1000 * 15 - ) { + if (now - this.last_ping <= 1000 * 15) { if (pong.now == null) { console.warn("pong must have a now field"); } else { @@ -61,20 +54,20 @@ export class TimeClient { // See the function server_time below; subtract this.clock_skew_ms from local // time to get a better estimate for server time. this.clock_skew_ms = - this.last_ping.valueOf() + - (this.last_pong.local.valueOf() - this.last_ping.valueOf()) / 2 - - this.last_pong.server.valueOf(); + this.last_ping + + (this.last_pong.local - this.last_ping) / 2 - + this.last_pong.server; set_local_storage("clock_skew", `${this.clock_skew_ms}`); } } - this.emit_latency(now.valueOf() - start.valueOf()); + this.emit_latency(now - start); if (!noLoop) { // periodically ping the server, to ensure clocks stay in sync. - setTimeout(this.ping.bind(this), this.ping_interval_ms); + setTimeout(this.ping, this.ping_interval_ms); } - } + }; private emit_latency(latency: number) { if (!window.document.hasFocus()) { @@ -110,11 +103,11 @@ export class TimeClient { const last = this.last_server_time; if (last != null && last >= t) { // That's annoying -- time is not marching forward... let's fake it until it does. - t = new Date(last.valueOf() + 1); + t = last + 1; } if ( this.last_pong != null && - Date.now() - this.last_pong.local.valueOf() < 5 * this.ping_interval_ms + Date.now() - this.last_pong.local < 5 * this.ping_interval_ms ) { // We have synced the clock **recently successfully**, so // we now ensure the time is increasing. @@ -125,10 +118,10 @@ export class TimeClient { } else { delete this.last_server_time; } - return t; + return new Date(t); } - private unskewed_server_time(): Date { + private unskewed_server_time(): number { // Add clock_skew_ms to our local time to get a better estimate of the actual time on the server. // This can help compensate in case the user's clock is wildly wrong, e.g., by several minutes, // or even hours due to totally wrong time (e.g. ignoring time zone), which is relevant for @@ -141,9 +134,9 @@ export class TimeClient { } } if (this.clock_skew_ms != null) { - return new Date(Date.now() - this.clock_skew_ms); + return Date.now() - this.clock_skew_ms; } else { - return new Date(); + return Date.now(); } } @@ -152,7 +145,7 @@ export class TimeClient { timeout?: number; // any ping that takes this long in seconds is considered a fail delay_ms?: number; // wait this long between doing pings log?: Function; // if set, use this to log output - }) { + }={}) { if (opts.packets == null) opts.packets = 20; if (opts.timeout == null) opts.timeout = 5; if (opts.delay_ms == null) opts.delay_ms = 200; @@ -164,15 +157,12 @@ export class TimeClient { */ const ping_times: number[] = []; const do_ping: (i: number) => Promise = async (i) => { - const t = new Date(); + const t = Date.now(); const heading = `${i}/${opts.packets}: `; let bar, mesg, pong, ping_time; try { - pong = await this.client.async_call({ - message: message.ping(), - timeout: opts.timeout, - }); - ping_time = Date.now() - t.valueOf(); + pong = await this.client.nats_client.hub.system.ping(); + ping_time = Date.now() - t; bar = ""; for (let j = 0; j <= Math.floor(ping_time / 10); j++) { bar += "*"; diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee index 96ac7d5d61..e33de54f4e 100644 --- a/src/packages/hub/client.coffee +++ b/src/packages/hub/client.coffee @@ -604,9 +604,6 @@ class exports.Client extends EventEmitter dbg("ignoring all further messages from old client=#{@id}") @_ignore_client = true - mesg_ping: (mesg) => - @push_to_client(message.pong(id:mesg.id, now:new Date())) - mesg_sign_in: (mesg) => sign_in.sign_in client : @ diff --git a/src/packages/nats/api/db.ts b/src/packages/nats/api/db.ts new file mode 100644 index 0000000000..0647b801c3 --- /dev/null +++ b/src/packages/nats/api/db.ts @@ -0,0 +1,14 @@ +import { authFirst } from "./util"; + +export const db = { + userQuery: authFirst, +}; + +export interface DB { + userQuery: (opts: { + project_id?: string; + account_id?: string; + query: any; + options?: any[]; + }) => Promise; +} diff --git a/src/packages/nats/api/index.ts b/src/packages/nats/api/index.ts index b4a9028b6c..1efa9164f8 100644 --- a/src/packages/nats/api/index.ts +++ b/src/packages/nats/api/index.ts @@ -1,53 +1,19 @@ -import type { Customize } from "@cocalc/util/db-schema/server-settings"; import { isValidUUID } from "@cocalc/util/misc"; +import { type Purchases, purchases } from "./purchases"; +import { type System, system } from "./system"; +import { type DB, db } from "./db"; export interface HubApi { - system: { - getCustomize: (fields?: string[]) => Promise; - }; - - db: { - userQuery: (opts: { - project_id?: string; - account_id?: string; - query: any; - options?: any[]; - }) => Promise; - }; - - purchases: { - getBalance: ({ account_id }) => Promise; - getMinBalance: (account_id) => Promise; - }; + system: System; + db: DB; + purchases: Purchases; } -const authFirst = ({ args, account_id, project_id }) => { - if (args[0] == null) { - args[0] = {} as any; - } - if (account_id) { - args[0].account_id = account_id; - } else if (project_id) { - args[0].project_id = project_id; - } - return args; -}; - -const noAuth = ({ args }) => args; const HubApiStructure = { - system: { - getCustomize: noAuth, - }, - db: { - userQuery: authFirst, - }, - purchases: { - getBalance: ({ account_id }) => { - return [{ account_id }]; - }, - getMinBalance: ({ account_id }) => [account_id], - }, + system, + db, + purchases, } as const; export function transformArgs({ name, args, account_id, project_id }) { diff --git a/src/packages/nats/api/purchases.ts b/src/packages/nats/api/purchases.ts new file mode 100644 index 0000000000..697160717f --- /dev/null +++ b/src/packages/nats/api/purchases.ts @@ -0,0 +1,11 @@ +export interface Purchases { + getBalance: ({ account_id }) => Promise; + getMinBalance: (account_id) => Promise; +} + +export const purchases = { + getBalance: ({ account_id }) => { + return [{ account_id }]; + }, + getMinBalance: ({ account_id }) => [account_id], +}; diff --git a/src/packages/nats/api/system.ts b/src/packages/nats/api/system.ts new file mode 100644 index 0000000000..26e5449610 --- /dev/null +++ b/src/packages/nats/api/system.ts @@ -0,0 +1,12 @@ +import { noAuth } from "./util"; +import type { Customize } from "@cocalc/util/db-schema/server-settings"; + +export const system = { + getCustomize: noAuth, + ping: noAuth, +}; + +export interface System { + getCustomize: (fields?: string[]) => Promise; + ping: () => { now: number }; +} diff --git a/src/packages/nats/api/util.ts b/src/packages/nats/api/util.ts new file mode 100644 index 0000000000..8c3724536f --- /dev/null +++ b/src/packages/nats/api/util.ts @@ -0,0 +1,13 @@ +export const authFirst = ({ args, account_id, project_id }) => { + if (args[0] == null) { + args[0] = {} as any; + } + if (account_id) { + args[0].account_id = account_id; + } else if (project_id) { + args[0].project_id = project_id; + } + return args; +}; + +export const noAuth = ({ args }) => args; diff --git a/src/packages/server/nats/api/db.ts b/src/packages/server/nats/api/db.ts new file mode 100644 index 0000000000..6863fd9e01 --- /dev/null +++ b/src/packages/server/nats/api/db.ts @@ -0,0 +1,3 @@ +import userQuery from "@cocalc/database/user-query"; + +export { userQuery }; diff --git a/src/packages/server/nats/api.ts b/src/packages/server/nats/api/index.ts similarity index 83% rename from src/packages/server/nats/api.ts rename to src/packages/server/nats/api/index.ts index b5b025d161..4708e8d5eb 100644 --- a/src/packages/server/nats/api.ts +++ b/src/packages/server/nats/api/index.ts @@ -27,16 +27,18 @@ To view all requests (and replies) in realtime: import { JSONCodec } from "nats"; import getLogger from "@cocalc/backend/logger"; import { type HubApi, getUserId, transformArgs } from "@cocalc/nats/api/index"; +import { getConnection } from "@cocalc/backend/nats"; const logger = getLogger("server:nats:api"); const jc = JSONCodec(); -export async function initAPI(nc) { +export async function initAPI() { const subject = "hub.*.*.api"; logger.debug(`initAPI -- subject='${subject}', options=`, { queue: "0", }); + const nc = await getConnection(); const sub = nc.subscribe(subject, { queue: "0" }); for await (const mesg of sub) { handleApiRequest(mesg); @@ -44,7 +46,6 @@ export async function initAPI(nc) { } async function handleApiRequest(mesg) { - console.log({ subject: mesg.subject }); let resp; try { const { account_id, project_id } = getUserId(mesg.subject); @@ -63,22 +64,14 @@ async function handleApiRequest(mesg) { mesg.respond(jc.encode(resp)); } -import userQuery from "@cocalc/database/user-query"; -import getCustomize from "@cocalc/database/settings/customize"; -import getBalance from "@cocalc/server/purchases/get-balance"; -import getMinBalance from "@cocalc/server/purchases/get-min-balance"; +import * as purchases from "./purchases"; +import * as db from "./db"; +import * as system from "./system"; -const hubApi: HubApi = { - system: { - getCustomize, - }, - db: { - userQuery, - }, - purchases: { - getBalance, - getMinBalance, - }, +export const hubApi: HubApi = { + system, + db, + purchases, }; async function getResponse({ name, args, account_id, project_id }) { diff --git a/src/packages/server/nats/api/purchases.ts b/src/packages/server/nats/api/purchases.ts new file mode 100644 index 0000000000..38831edb5e --- /dev/null +++ b/src/packages/server/nats/api/purchases.ts @@ -0,0 +1,4 @@ +import getBalance from "@cocalc/server/purchases/get-balance"; +import getMinBalance from "@cocalc/server/purchases/get-min-balance"; + +export { getBalance, getMinBalance }; diff --git a/src/packages/server/nats/api/system.ts b/src/packages/server/nats/api/system.ts new file mode 100644 index 0000000000..770441478c --- /dev/null +++ b/src/packages/server/nats/api/system.ts @@ -0,0 +1,6 @@ +import getCustomize from "@cocalc/database/settings/customize"; +export { getCustomize }; + +export function ping() { + return { now: Date.now() }; +} diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts index 1f5ea2047e..2ccc38944c 100644 --- a/src/packages/server/nats/index.ts +++ b/src/packages/server/nats/index.ts @@ -1,12 +1,9 @@ import getLogger from "@cocalc/backend/logger"; import { initAPI } from "./api"; -import { getConnection } from "@cocalc/backend/nats"; const logger = getLogger("server:nats"); export default async function initNatsServer() { logger.debug("initializing nats cocalc hub server"); - const nc = await getConnection(); - logger.debug(`connected to ${nc.getServer()}`); - initAPI(nc); + initAPI(); } diff --git a/src/packages/server/package.json b/src/packages/server/package.json index 0ac34057dc..d63a5963a6 100644 --- a/src/packages/server/package.json +++ b/src/packages/server/package.json @@ -13,6 +13,7 @@ "./database/*": "./dist/database/*.js", "./mentions/*": "./dist/mentions/*.js", "./nats": "./dist/nats/index.js", + "./nats/api": "./dist/nats/api/index.js", "./purchases/*": "./dist/purchases/*.js", "./stripe/*": "./dist/stripe/*.js", "./licenses/purchase": "./dist/licenses/purchase/index.js", From 89e230c459575c726ec67ad94a1fbb1dec7e2e67 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 28 Jan 2025 02:25:03 +0000 Subject: [PATCH 053/281] proper pnpm-lock update --- src/packages/pnpm-lock.yaml | 546 +++++++++++++++++------------------- 1 file changed, 263 insertions(+), 283 deletions(-) diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index b942f7789e..b770b328f2 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -31,10 +31,10 @@ importers: version: 5.0.0 jest: specifier: ^29.7.0 - version: 29.7.0(@types/node@18.19.55) + version: 29.7.0(@types/node@18.19.74) ts-jest: specifier: ^29.2.3 - version: 29.2.5(@babel/core@7.25.8)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.8))(jest@29.7.0(@types/node@18.19.55))(typescript@5.7.3) + version: 29.2.5(@babel/core@7.25.8)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.8))(jest@29.7.0(@types/node@18.19.74))(typescript@5.7.3) typescript: specifier: ^5.7.3 version: 5.7.3 @@ -103,6 +103,9 @@ importers: lru-cache: specifier: ^7.18.3 version: 7.18.3 + nats: + specifier: ^2.29.1 + version: 2.29.1 password-hash: specifier: ^1.2.2 version: 1.2.2 @@ -168,6 +171,9 @@ importers: '@cocalc/database': specifier: workspace:* version: 'link:' + '@cocalc/nats': + specifier: workspace:* + version: link:../nats '@cocalc/util': specifier: workspace:* version: link:../util @@ -204,6 +210,9 @@ importers: lru-cache: specifier: ^7.18.3 version: 7.18.3 + nats: + specifier: ^2.29.1 + version: 2.29.1 node-fetch: specifier: 2.6.7 version: 2.6.7(encoding@0.1.13) @@ -262,6 +271,9 @@ importers: '@cocalc/local-storage-lru': specifier: ^2.4.3 version: 2.5.0 + '@cocalc/nats': + specifier: workspace:* + version: link:../nats '@cocalc/sync': specifier: workspace:* version: link:../sync @@ -301,6 +313,12 @@ importers: '@microlink/react-json-view': specifier: ^1.23.3 version: 1.23.3(@types/react@18.3.10)(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@nats-io/jetstream': + specifier: 3.0.0-36 + version: 3.0.0-36 + '@nats-io/kv': + specifier: 3.0.0-30 + version: 3.0.0-30 '@orama/orama': specifier: 3.0.0-rc-3 version: 3.0.0-rc-3 @@ -511,6 +529,9 @@ importers: mermaid: specifier: ^11.3.0 version: 11.3.0 + nats.ws: + specifier: ^1.30.0 + version: 1.30.1 node-forge: specifier: ^1.0.0 version: 1.3.1 @@ -582,16 +603,13 @@ importers: version: 2.6.0(plotly.js@2.35.2(@rspack/core@1.2.2(@swc/helpers@0.5.15))(mapbox-gl@3.9.4)(webpack@5.97.1))(react@18.3.1) react-redux: specifier: ^8.0.5 - version: 8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@5.0.1) + version: 8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@4.2.1) react-timeago: specifier: ^7.2.0 version: 7.2.0(react@18.3.1) react-virtuoso: specifier: ^4.9.0 version: 4.10.4(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - sha1: - specifier: ^1.1.1 - version: 1.1.1 shallowequal: specifier: ^1.1.0 version: 1.1.0 @@ -833,6 +851,9 @@ importers: ms: specifier: 2.1.2 version: 2.1.2 + nats: + specifier: ^2.29.1 + version: 2.29.1 next: specifier: 14.2.22 version: 14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.83.4) @@ -1028,6 +1049,43 @@ importers: specifier: ^18.16.14 version: 18.19.50 + nats: + dependencies: + '@cocalc/nats': + specifier: workspace:* + version: 'link:' + '@cocalc/util': + specifier: workspace:* + version: link:../util + '@nats-io/jetstream': + specifier: 3.0.0-36 + version: 3.0.0-36 + '@nats-io/kv': + specifier: 3.0.0-30 + version: 3.0.0-30 + events: + specifier: 3.3.0 + version: 3.3.0 + json-stable-stringify: + specifier: ^1.0.1 + version: 1.1.1 + lodash: + specifier: ^4.17.21 + version: 4.17.21 + sha1: + specifier: ^1.1.1 + version: 1.1.1 + devDependencies: + '@types/json-stable-stringify': + specifier: ^1.0.32 + version: 1.0.36 + '@types/lodash': + specifier: ^4.14.202 + version: 4.17.9 + '@types/node': + specifier: ^18.16.14 + version: 18.19.74 + next: dependencies: '@ant-design/icons': @@ -1138,9 +1196,6 @@ importers: serve-index: specifier: ^1.9.1 version: 1.9.1 - sha1: - specifier: ^1.1.1 - version: 1.1.1 sharp: specifier: ^0.32.6 version: 0.32.6 @@ -1193,6 +1248,9 @@ importers: '@cocalc/jupyter': specifier: workspace:* version: link:../jupyter + '@cocalc/nats': + specifier: workspace:* + version: link:../nats '@cocalc/primus-multiplex': specifier: ^1.1.0 version: 1.1.0 @@ -1217,6 +1275,12 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util + '@nats-io/jetstream': + specifier: 3.0.0-36 + version: 3.0.0-36 + '@nats-io/kv': + specifier: 3.0.0-30 + version: 3.0.0-30 '@nteract/messaging': specifier: ^7.0.20 version: 7.0.20 @@ -1283,6 +1347,12 @@ importers: lru-cache: specifier: ^7.18.3 version: 7.18.3 + nats: + specifier: ^2.29.1 + version: 2.29.1 + node-pty: + specifier: ^1.0.0 + version: 1.0.0 pidusage: specifier: ^1.2.0 version: 1.2.0 @@ -1347,6 +1417,9 @@ importers: '@cocalc/gcloud-pricing-calculator': specifier: ^1.14.0 version: 1.14.0 + '@cocalc/nats': + specifier: workspace:* + version: link:../nats '@cocalc/server': specifier: workspace:* version: 'link:' @@ -1382,7 +1455,7 @@ importers: version: 0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) '@langchain/community': specifier: ^0.3.24 - version: 0.3.24(@browserbasehq/sdk@2.0.0(encoding@0.1.13))(@browserbasehq/stagehand@1.10.1(@playwright/test@1.50.0)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.3.2(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@11.8.1)(cheerio@1.0.0)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.1)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.1)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.1)(playwright@1.50.0)(ws@8.18.0) + version: 0.3.24(@browserbasehq/sdk@2.0.0(encoding@0.1.13))(@browserbasehq/stagehand@1.10.1(@playwright/test@1.50.0)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.3.2(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@8.7.0)(cheerio@1.0.0-rc.10)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.1)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.1)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.1)(playwright@1.50.0)(ws@8.18.0) '@langchain/core': specifier: ^0.3.30 version: 0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) @@ -1524,6 +1597,9 @@ importers: nanoid: specifier: ^3.3.8 version: 3.3.8 + nats: + specifier: ^2.29.1 + version: 2.29.1 node-zendesk: specifier: ^5.0.13 version: 5.0.13(encoding@0.1.13) @@ -1835,9 +1911,6 @@ importers: lodash: specifier: ^4.17.21 version: 4.17.21 - sha1: - specifier: ^1.1.1 - version: 1.1.1 devDependencies: '@types/node': specifier: ^18.16.14 @@ -3848,6 +3921,23 @@ packages: '@module-federation/webpack-bundler-runtime@0.8.4': resolution: {integrity: sha512-HggROJhvHPUX7uqBD/XlajGygMNM1DG0+4OAkk8MBQe4a18QzrRNzZt6XQbRTSG4OaEoyRWhQHvYD3Yps405tQ==} + '@nats-io/jetstream@3.0.0-36': + resolution: {integrity: sha512-95+ftM+lXcUEwCl3ctzpJZCXiBUwRuSUUWmd4X8jJlNVZc/LUHH+3cxX+nC42sLHD+9zvIaMKPsyI0Ltbp+BSg==} + + '@nats-io/kv@3.0.0-30': + resolution: {integrity: sha512-rg3y1/v4gTP6PATpAOK32cXgPfez8PZqC9r39vqT7vVIGUNKERFcX8JdqC+50B97ovqvlTSvItR9IsRNdYAIZg==} + + '@nats-io/nats-core@3.0.0-49': + resolution: {integrity: sha512-Xe7LjCdhtL4pXk2czwUE8Y1elTy/zo3ZzpoIwOO+/uJPughEsSxCpqygPrDqWcOG2uWVB9G1wxjg8r0Y9StovQ==} + + '@nats-io/nkeys@2.0.2': + resolution: {integrity: sha512-0JTyVl9P+UJyjUBDWP9589TuUKXJQ8tDkVRgi02X/MMzW997+4FykirvZEkIe6ZAhiLIBN+NpN8ULMMt6mDrbA==} + engines: {node: '>=18.0.0'} + + '@nats-io/nuid@2.0.3': + resolution: {integrity: sha512-TpA3HEBna/qMVudy+3HZr5M3mo/L1JPofpVT4t0HkFGkz2Cn9wrlrQC8tvR8Md5Oa9//GtGG26eN0qEWF5Vqew==} + engines: {node: '>= 18.x'} + '@nestjs/axios@3.0.3': resolution: {integrity: sha512-h6TCn3yJwD6OKqqqfmtRS5Zo4E46Ip2n+gK1sqwzNBC+qxQ9xpCu+ODVRFur6V3alHSCSBxb3nNtt73VEdluyA==} peerDependencies: @@ -4708,18 +4798,9 @@ packages: '@types/node@18.19.50': resolution: {integrity: sha512-xonK+NRrMBRtkL1hVCc3G+uXtjh1Al4opBLjqVmipe5ZAaBYWW6cNAiBVZ1BvmkBhep698rP3UM3aRAdSALuhg==} - '@types/node@18.19.55': - resolution: {integrity: sha512-zzw5Vw52205Zr/nmErSEkN5FLqXPuKX/k5d1D7RKHATGqU7y6YfX9QxZraUzUrFGqH6XzOzG196BC35ltJC4Cw==} - - '@types/node@18.19.64': - resolution: {integrity: sha512-955mDqvO2vFf/oL7V3WiUtiz+BugyX8uVbaT2H8oj3+8dRyH2FLiNdowe7eNqRM7IOIZvzDH76EoAT+gwm6aIQ==} - '@types/node@18.19.74': resolution: {integrity: sha512-HMwEkkifei3L605gFdV+/UwtpxP6JSzM+xFk2Ia6DNFSwSVBRh9qp5Tgf4lNFOMfPVuU0WnkcWpXZpgn5ufO4A==} - '@types/node@22.12.0': - resolution: {integrity: sha512-Fll2FZ1riMjNmlmJOdAyY5pUbkftXslB5DgEzlIuNaiWhXd00FhWxVC/r4yV/4wBb9JfImTu+jiSvXTkJ7F/gA==} - '@types/node@9.6.61': resolution: {integrity: sha512-/aKAdg5c8n468cYLy2eQrcR5k6chlbNwZNGUj3TboyPa2hcO2QAJcfymlqPzMiRj8B6nYKXjzQz36minFE0RwQ==} @@ -5468,9 +5549,6 @@ packages: bcryptjs@2.4.3: resolution: {integrity: sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==} - better-sqlite3@11.8.1: - resolution: {integrity: sha512-9BxNaBkblMjhJW8sMRZxnxVTRgbRmssZW0Oxc1MPBTfiR+WW21e2Mk4qu8CzrcZb1LwPCnFsfDEzq+SNcBU8eg==} - better-sqlite3@8.7.0: resolution: {integrity: sha512-99jZU4le+f3G6aIl6PmmV0cxUIWqKieHxsiF7G34CVFiE+/UabpYqkU0NJIkY/96mQKikHeBjtR27vFfs5JpEw==} @@ -5723,10 +5801,6 @@ packages: cheerio-select@2.1.0: resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} - cheerio@1.0.0: - resolution: {integrity: sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==} - engines: {node: '>=18.17'} - cheerio@1.0.0-rc.10: resolution: {integrity: sha512-g0J0q/O6mW8z5zxQ3A8E8J1hUgp4SMOvEoW/x84OwyHKe/Zccz83PVT4y5Crcr530FV6NgmKI1qvGTKVl9XXVw==} engines: {node: '>= 6'} @@ -6865,9 +6939,6 @@ packages: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} - encoding-sniffer@0.2.0: - resolution: {integrity: sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==} - encoding@0.1.13: resolution: {integrity: sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==} @@ -7915,9 +7986,6 @@ packages: htmlparser2@8.0.2: resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} - htmlparser2@9.1.0: - resolution: {integrity: sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==} - htmlparser@1.7.7: resolution: {integrity: sha512-zpK66ifkT0fauyFh2Mulrq4AqGTucxGtOhZ8OjkbSfcCpkqQEI8qRkY0tSQSJNAQ4HUZkgWaU4fK4EH6SVH9PQ==} engines: {node: '>=0.1.33'} @@ -9434,12 +9502,16 @@ packages: napi-build-utils@1.0.2: resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} - napi-build-utils@2.0.0: - resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} - native-promise-only@0.8.1: resolution: {integrity: sha512-zkVhZUA3y8mbz652WrL5x0fB0ehrBkulWT3TomAQ9iDtyXZvzKeEA6GPxAItBYeNYl5yngKRX612qHOhvMkDeg==} + nats.ws@1.30.1: + resolution: {integrity: sha512-OVufQJrYuzt9Gugb4R/kCEM9R03dH2kMZ0Lt59LRL3a0+8bJjxodAxUOh952DiuPTquwGg8JIF8h3BKkTF5mJA==} + + nats@2.29.1: + resolution: {integrity: sha512-OHVsxrQCITTdMKG3So0jhtnBd5jS2u1xpS91UCws7VklsaCbctwg5vT/8lYpVldPW0x3aHGF8uuAoMfCoJy7Sg==} + engines: {node: '>= 14.0.0'} + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -9506,6 +9578,10 @@ packages: nise@1.5.3: resolution: {integrity: sha512-Ymbac/94xeIrMf59REBPOv0thr+CJVFMhrlAkW/gjCIE58BGQdCj0x7KRCb3yz+Ga2Rz3E9XXSvUyyxqqhjQAQ==} + nkeys.js@1.1.0: + resolution: {integrity: sha512-tB/a0shZL5UZWSwsoeyqfTszONTt4k2YS0tuQioMOD180+MbombYVgzDUYHlx+gejYK6rgf08n/2Df99WY0Sxg==} + engines: {node: '>=10.0.0'} + no-case@3.0.4: resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} @@ -9513,10 +9589,6 @@ packages: resolution: {integrity: sha512-bLn/fU/ALVBE9wj+p4Y21ZJWYFjUXLXPi/IewyLZkx3ApxKDNBWCKdReeKOtD8dWpOdDCeMyLh6ZewzcLsG2Nw==} engines: {node: '>=10'} - node-abi@3.73.0: - resolution: {integrity: sha512-z8iYzQGBu35ZkTQ9mtR8RqugJZ9RCLn8fv3d7LsgDBzOijGQP3RdKTX4LA7LXw03ZhU5z0l4xfhIMgSES31+cg==} - engines: {node: '>=10'} - node-addon-api@6.1.0: resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} @@ -9880,9 +9952,6 @@ packages: parse5-htmlparser2-tree-adapter@7.1.0: resolution: {integrity: sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==} - parse5-parser-stream@7.1.2: - resolution: {integrity: sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==} - parse5@6.0.1: resolution: {integrity: sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==} @@ -10287,11 +10356,6 @@ packages: engines: {node: '>=10'} hasBin: true - prebuild-install@7.1.3: - resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} - engines: {node: '>=10'} - hasBin: true - precond@0.2.3: resolution: {integrity: sha512-QCYG84SgGyGzqJ/vlMsxeXd/pgL/I94ixdNFyh1PusWmTCyVfPJjZ1K1jvHtsbfnXQs2TSkEP2fR7QiMZAnKFQ==} engines: {node: '>= 0.6'} @@ -10961,9 +11025,6 @@ packages: redux@4.2.1: resolution: {integrity: sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==} - redux@5.0.1: - resolution: {integrity: sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==} - reflect-metadata@0.1.13: resolution: {integrity: sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==} @@ -11773,9 +11834,6 @@ packages: tar-fs@2.1.1: resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==} - tar-fs@2.1.2: - resolution: {integrity: sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==} - tar-fs@3.0.6: resolution: {integrity: sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==} @@ -12032,6 +12090,9 @@ packages: tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + tweetnacl@1.0.3: + resolution: {integrity: sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -12156,13 +12217,6 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@6.20.0: - resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} - - undici@6.21.1: - resolution: {integrity: sha512-q/1rj5D0/zayJB2FraXdaWxbhWiNKDvu8naDT2dl1yTlvJp4BLtOcp2a5BvgGNQpYYJzau7tf1WgKv3b+7mqpQ==} - engines: {node: '>=18.17'} - unicorn-magic@0.1.0: resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==} engines: {node: '>=18'} @@ -12530,14 +12584,6 @@ packages: webworkify@1.5.0: resolution: {integrity: sha512-AMcUeyXAhbACL8S2hqqdqOLqvJ8ylmIbNwUIqQujRSouf4+eUFaXbG6F1Rbu+srlJMmxQWsiU7mOJi0nMBfM1g==} - whatwg-encoding@3.1.1: - resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} - engines: {node: '>=18'} - - whatwg-mimetype@4.0.0: - resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} - engines: {node: '>=18'} - whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} @@ -12908,13 +12954,13 @@ snapshots: agentkeepalive: 4.6.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.6.7(encoding@0.1.13) transitivePeerDependencies: - encoding '@anthropic-ai/sdk@0.32.1(encoding@0.1.13)': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -13586,7 +13632,7 @@ snapshots: agentkeepalive: 4.6.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.6.7(encoding@0.1.13) transitivePeerDependencies: - encoding @@ -14220,7 +14266,7 @@ snapshots: '@jest/console@29.7.0': dependencies: '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.74 chalk: 4.1.2 jest-message-util: 29.7.0 jest-util: 29.7.0 @@ -14233,14 +14279,14 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.55 + '@types/node': 18.19.74 ansi-escapes: 4.3.2 chalk: 4.1.2 ci-info: 3.9.0 exit: 0.1.2 graceful-fs: 4.2.11 jest-changed-files: 29.7.0 - jest-config: 29.7.0(@types/node@18.19.55) + jest-config: 29.7.0(@types/node@18.19.74) jest-haste-map: 29.7.0 jest-message-util: 29.7.0 jest-regex-util: 29.6.3 @@ -14265,7 +14311,7 @@ snapshots: dependencies: '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.74 jest-mock: 29.7.0 '@jest/expect-utils@29.7.0': @@ -14283,7 +14329,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@sinonjs/fake-timers': 10.3.0 - '@types/node': 18.19.64 + '@types/node': 18.19.74 jest-message-util: 29.7.0 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -14305,7 +14351,7 @@ snapshots: '@jest/transform': 29.7.0 '@jest/types': 29.6.3 '@jridgewell/trace-mapping': 0.3.25 - '@types/node': 18.19.64 + '@types/node': 18.19.74 chalk: 4.1.2 collect-v8-coverage: 1.0.2 exit: 0.1.2 @@ -14374,7 +14420,7 @@ snapshots: dependencies: '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.1 - '@types/node': 18.19.50 + '@types/node': 18.19.74 '@types/yargs': 15.0.19 chalk: 4.1.2 @@ -14383,7 +14429,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.1 - '@types/node': 18.19.55 + '@types/node': 18.19.74 '@types/yargs': 17.0.24 chalk: 4.1.2 @@ -14582,7 +14628,7 @@ snapshots: transitivePeerDependencies: - encoding - ? '@langchain/community@0.3.24(@browserbasehq/sdk@2.0.0(encoding@0.1.13))(@browserbasehq/stagehand@1.10.1(@playwright/test@1.50.0)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.3.2(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@11.8.1)(cheerio@1.0.0)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.1)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.1)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.1)(playwright@1.50.0)(ws@8.18.0)' + ? '@langchain/community@0.3.24(@browserbasehq/sdk@2.0.0(encoding@0.1.13))(@browserbasehq/stagehand@1.10.1(@playwright/test@1.50.0)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.3.2(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@8.7.0)(cheerio@1.0.0-rc.10)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.1)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.1)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.1)(playwright@1.50.0)(ws@8.18.0)' : dependencies: '@browserbasehq/stagehand': 1.10.1(@playwright/test@1.50.0)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8) '@ibm-cloud/watsonx-ai': 1.3.2(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))) @@ -14593,7 +14639,7 @@ snapshots: flat: 5.0.2 ibm-cloud-sdk-core: 5.1.1 js-yaml: 4.1.0 - langchain: 0.3.11(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(cheerio@1.0.0)(encoding@0.1.13)(handlebars@4.7.8)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) + langchain: 0.3.11(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(cheerio@1.0.0-rc.10)(encoding@0.1.13)(handlebars@4.7.8)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) langsmith: 0.2.15(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) openai: 4.78.1(encoding@0.1.13)(zod@3.23.8) uuid: 10.0.0 @@ -14603,8 +14649,8 @@ snapshots: '@browserbasehq/sdk': 2.0.0(encoding@0.1.13) '@google-ai/generativelanguage': 2.7.0(encoding@0.1.13) '@google-cloud/storage': 7.13.0(encoding@0.1.13) - better-sqlite3: 11.8.1 - cheerio: 1.0.0 + better-sqlite3: 8.7.0 + cheerio: 1.0.0-rc.10 d3-dsv: 3.0.1 fast-xml-parser: 4.5.1 google-auth-library: 9.14.1(encoding@0.1.13) @@ -14887,6 +14933,26 @@ snapshots: '@module-federation/sdk': 0.8.4 optional: true + '@nats-io/jetstream@3.0.0-36': + dependencies: + '@nats-io/nats-core': 3.0.0-49 + + '@nats-io/kv@3.0.0-30': + dependencies: + '@nats-io/jetstream': 3.0.0-36 + '@nats-io/nats-core': 3.0.0-49 + + '@nats-io/nats-core@3.0.0-49': + dependencies: + '@nats-io/nkeys': 2.0.2 + '@nats-io/nuid': 2.0.3 + + '@nats-io/nkeys@2.0.2': + dependencies: + tweetnacl: 1.0.3 + + '@nats-io/nuid@2.0.3': {} + '@nestjs/axios@3.0.3(@nestjs/common@10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1))(axios@1.7.4)(rxjs@7.8.1)': dependencies: '@nestjs/common': 10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1) @@ -15629,11 +15695,11 @@ snapshots: '@types/body-parser@1.19.5': dependencies: '@types/connect': 3.4.35 - '@types/node': 18.19.50 + '@types/node': 18.19.74 '@types/bonjour@3.5.13': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/caseless@0.12.5': {} @@ -15646,11 +15712,11 @@ snapshots: '@types/connect-history-api-fallback@1.5.4': dependencies: '@types/express-serve-static-core': 5.0.1 - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/connect@3.4.35': dependencies: - '@types/node': 18.19.55 + '@types/node': 18.19.74 '@types/cookie@0.3.3': {} @@ -15683,14 +15749,14 @@ snapshots: '@types/express-serve-static-core@4.19.0': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/qs': 6.9.17 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 '@types/express-serve-static-core@5.0.1': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/qs': 6.9.17 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 @@ -15728,11 +15794,11 @@ snapshots: '@types/glob@7.2.0': dependencies: '@types/minimatch': 5.1.2 - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/graceful-fs@4.1.9': dependencies: - '@types/node': 18.19.55 + '@types/node': 18.19.74 '@types/hast@2.3.10': dependencies: @@ -15782,11 +15848,11 @@ snapshots: '@types/keyv@3.1.4': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/ldapjs@2.2.5': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/linkify-it@5.0.0': {} @@ -15796,7 +15862,7 @@ snapshots: '@types/lz4@0.6.4': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.74 '@types/mapbox__point-geometry@0.1.4': {} @@ -15833,7 +15899,7 @@ snapshots: '@types/node-fetch@2.6.11': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 form-data: 4.0.0 '@types/node-fetch@2.6.12': @@ -15843,11 +15909,11 @@ snapshots: '@types/node-forge@1.3.11': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/node-zendesk@2.0.15': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.74 '@types/node@10.14.22': {} @@ -15855,33 +15921,21 @@ snapshots: dependencies: undici-types: 5.26.5 - '@types/node@18.19.55': - dependencies: - undici-types: 5.26.5 - - '@types/node@18.19.64': - dependencies: - undici-types: 5.26.5 - '@types/node@18.19.74': dependencies: undici-types: 5.26.5 - '@types/node@22.12.0': - dependencies: - undici-types: 6.20.0 - '@types/node@9.6.61': {} '@types/nodemailer@6.4.16': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.74 '@types/normalize-package-data@2.4.1': {} '@types/oauth@0.9.1': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/parse5@6.0.3': {} @@ -15949,13 +16003,13 @@ snapshots: '@types/request@2.48.12': dependencies: '@types/caseless': 0.12.5 - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/tough-cookie': 4.0.5 form-data: 2.5.1 '@types/responselike@1.0.3': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/retry@0.12.0': {} @@ -15972,7 +16026,7 @@ snapshots: '@types/send@0.17.4': dependencies: '@types/mime': 1.3.5 - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/serve-index@1.9.4': dependencies: @@ -15981,19 +16035,19 @@ snapshots: '@types/serve-static@1.15.0': dependencies: '@types/mime': 3.0.1 - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/serve-static@1.15.7': dependencies: '@types/http-errors': 2.0.4 - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/send': 0.17.4 '@types/sizzle@2.3.3': {} '@types/sockjs@0.3.36': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/stack-utils@2.0.3': {} @@ -16024,20 +16078,20 @@ snapshots: '@types/ws@8.5.13': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/xml-crypto@1.4.6': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 xpath: 0.0.27 '@types/xml-encryption@1.2.4': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/xml2js@0.4.14': dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/yargs-parser@21.0.0': {} @@ -16499,7 +16553,7 @@ snapshots: define-properties: 1.2.1 es-abstract: 1.23.3 es-object-atoms: 1.0.0 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 is-string: 1.0.7 array-keyed-map@2.1.3: {} @@ -16555,7 +16609,7 @@ snapshots: define-properties: 1.2.1 es-abstract: 1.23.3 es-errors: 1.3.0 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 is-array-buffer: 3.0.4 is-shared-array-buffer: 1.0.3 @@ -16805,12 +16859,6 @@ snapshots: bcryptjs@2.4.3: {} - better-sqlite3@11.8.1: - dependencies: - bindings: 1.5.0 - prebuild-install: 7.1.3 - optional: true - better-sqlite3@8.7.0: dependencies: bindings: 1.5.0 @@ -17101,21 +17149,6 @@ snapshots: domhandler: 5.0.3 domutils: 3.2.2 - cheerio@1.0.0: - dependencies: - cheerio-select: 2.1.0 - dom-serializer: 2.0.0 - domhandler: 5.0.3 - domutils: 3.2.2 - encoding-sniffer: 0.2.0 - htmlparser2: 9.1.0 - parse5: 7.2.1 - parse5-htmlparser2-tree-adapter: 7.1.0 - parse5-parser-stream: 7.1.2 - undici: 6.21.1 - whatwg-mimetype: 4.0.0 - optional: true - cheerio@1.0.0-rc.10: dependencies: cheerio-select: 1.6.0 @@ -17545,13 +17578,13 @@ snapshots: - supports-color - ts-node - create-jest@29.7.0(@types/node@18.19.55): + create-jest@29.7.0(@types/node@18.19.74): dependencies: '@jest/types': 29.6.3 chalk: 4.1.2 exit: 0.1.2 graceful-fs: 4.2.11 - jest-config: 29.7.0(@types/node@18.19.55) + jest-config: 29.7.0(@types/node@18.19.74) jest-util: 29.7.0 prompts: 2.4.2 transitivePeerDependencies: @@ -18110,9 +18143,9 @@ snapshots: define-data-property@1.1.4: dependencies: - es-define-property: 1.0.0 + es-define-property: 1.0.1 es-errors: 1.3.0 - gopd: 1.0.1 + gopd: 1.2.0 define-lazy-prop@3.0.0: {} @@ -18393,12 +18426,6 @@ snapshots: encodeurl@2.0.0: {} - encoding-sniffer@0.2.0: - dependencies: - iconv-lite: 0.6.3 - whatwg-encoding: 3.1.1 - optional: true - encoding@0.1.13: dependencies: iconv-lite: 0.6.3 @@ -18444,19 +18471,19 @@ snapshots: data-view-buffer: 1.0.1 data-view-byte-length: 1.0.1 data-view-byte-offset: 1.0.0 - es-define-property: 1.0.0 + es-define-property: 1.0.1 es-errors: 1.3.0 - es-object-atoms: 1.0.0 + es-object-atoms: 1.1.1 es-set-tostringtag: 2.0.3 es-to-primitive: 1.2.1 function.prototype.name: 1.1.6 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 get-symbol-description: 1.0.2 globalthis: 1.0.4 - gopd: 1.0.1 + gopd: 1.2.0 has-property-descriptors: 1.0.2 has-proto: 1.0.3 - has-symbols: 1.0.3 + has-symbols: 1.1.0 hasown: 2.0.2 internal-slot: 1.0.7 is-array-buffer: 3.0.4 @@ -18488,7 +18515,7 @@ snapshots: es-define-property@1.0.0: dependencies: - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 es-define-property@1.0.1: {} @@ -18502,7 +18529,7 @@ snapshots: es-errors: 1.3.0 es-set-tostringtag: 2.0.3 function-bind: 1.1.2 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 globalthis: 1.0.4 has-property-descriptors: 1.0.2 has-proto: 1.0.3 @@ -18523,7 +18550,7 @@ snapshots: es-set-tostringtag@2.0.3: dependencies: - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 has-tostringtag: 1.0.2 hasown: 2.0.2 @@ -19215,7 +19242,7 @@ snapshots: es-errors: 1.3.0 function-bind: 1.1.2 has-proto: 1.0.3 - has-symbols: 1.0.3 + has-symbols: 1.1.0 hasown: 2.0.2 get-intrinsic@1.2.7: @@ -19256,7 +19283,7 @@ snapshots: dependencies: call-bind: 1.0.7 es-errors: 1.3.0 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 get-value@2.0.6: {} @@ -19365,7 +19392,7 @@ snapshots: globalthis@1.0.4: dependencies: define-properties: 1.2.1 - gopd: 1.0.1 + gopd: 1.2.0 globby@11.1.0: dependencies: @@ -19531,7 +19558,7 @@ snapshots: gopd@1.0.1: dependencies: - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 gopd@1.2.0: {} @@ -19616,7 +19643,7 @@ snapshots: has-property-descriptors@1.0.2: dependencies: - es-define-property: 1.0.0 + es-define-property: 1.0.1 has-proto@1.0.3: {} @@ -19626,11 +19653,11 @@ snapshots: has-tostringtag@1.0.0: dependencies: - has-symbols: 1.0.3 + has-symbols: 1.1.0 has-tostringtag@1.0.2: dependencies: - has-symbols: 1.0.3 + has-symbols: 1.1.0 has-unicode@2.0.1: optional: true @@ -19827,14 +19854,6 @@ snapshots: domutils: 3.2.2 entities: 4.5.0 - htmlparser2@9.1.0: - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - domutils: 3.2.2 - entities: 4.5.0 - optional: true - htmlparser@1.7.7: {} http-deceiver@1.2.7: {} @@ -20075,7 +20094,7 @@ snapshots: is-array-buffer@3.0.4: dependencies: call-bind: 1.0.7 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 is-arrayish@0.2.1: {} @@ -20246,14 +20265,14 @@ snapshots: is-symbol@1.0.4: dependencies: - has-symbols: 1.0.3 + has-symbols: 1.1.0 is-typed-array@1.1.10: dependencies: available-typed-arrays: 1.0.5 call-bind: 1.0.7 for-each: 0.3.3 - gopd: 1.0.1 + gopd: 1.2.0 has-tostringtag: 1.0.0 is-typed-array@1.1.13: @@ -20273,7 +20292,7 @@ snapshots: is-weakset@2.0.3: dependencies: call-bind: 1.0.7 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 is-what@3.14.1: {} @@ -20389,8 +20408,8 @@ snapshots: iterator.prototype@1.1.2: dependencies: define-properties: 1.2.1 - get-intrinsic: 1.2.4 - has-symbols: 1.0.3 + get-intrinsic: 1.2.7 + has-symbols: 1.1.0 reflect.getprototypeof: 1.0.6 set-function-name: 2.0.2 @@ -20419,7 +20438,7 @@ snapshots: '@jest/expect': 29.7.0 '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.74 chalk: 4.1.2 co: 4.6.0 dedent: 1.5.3 @@ -20458,16 +20477,16 @@ snapshots: - supports-color - ts-node - jest-cli@29.7.0(@types/node@18.19.55): + jest-cli@29.7.0(@types/node@18.19.74): dependencies: '@jest/core': 29.7.0 '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 chalk: 4.1.2 - create-jest: 29.7.0(@types/node@18.19.55) + create-jest: 29.7.0(@types/node@18.19.74) exit: 0.1.2 import-local: 3.2.0 - jest-config: 29.7.0(@types/node@18.19.55) + jest-config: 29.7.0(@types/node@18.19.74) jest-util: 29.7.0 jest-validate: 29.7.0 yargs: 17.7.2 @@ -20507,7 +20526,7 @@ snapshots: - babel-plugin-macros - supports-color - jest-config@29.7.0(@types/node@18.19.55): + jest-config@29.7.0(@types/node@18.19.74): dependencies: '@babel/core': 7.25.8 '@jest/test-sequencer': 29.7.0 @@ -20532,7 +20551,7 @@ snapshots: slash: 3.0.0 strip-json-comments: 3.1.1 optionalDependencies: - '@types/node': 18.19.55 + '@types/node': 18.19.74 transitivePeerDependencies: - babel-plugin-macros - supports-color @@ -20568,7 +20587,7 @@ snapshots: '@jest/environment': 29.7.0 '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.74 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -20580,7 +20599,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@types/graceful-fs': 4.1.9 - '@types/node': 18.19.64 + '@types/node': 18.19.74 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11 @@ -20638,7 +20657,7 @@ snapshots: jest-mock@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.74 jest-util: 29.7.0 jest-pnp-resolver@1.2.3(jest-resolve@29.7.0): @@ -20675,7 +20694,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.74 chalk: 4.1.2 emittery: 0.13.1 graceful-fs: 4.2.11 @@ -20703,7 +20722,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.74 chalk: 4.1.2 cjs-module-lexer: 1.2.3 collect-v8-coverage: 1.0.2 @@ -20749,7 +20768,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 18.19.55 + '@types/node': 18.19.74 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -20768,7 +20787,7 @@ snapshots: dependencies: '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 18.19.64 + '@types/node': 18.19.74 ansi-escapes: 4.3.2 chalk: 4.1.2 emittery: 0.13.1 @@ -20777,13 +20796,13 @@ snapshots: jest-worker@27.5.1: dependencies: - '@types/node': 22.12.0 + '@types/node': 18.19.74 merge-stream: 2.0.0 supports-color: 8.1.1 jest-worker@29.7.0: dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 jest-util: 29.7.0 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -20800,12 +20819,12 @@ snapshots: - supports-color - ts-node - jest@29.7.0(@types/node@18.19.55): + jest@29.7.0(@types/node@18.19.74): dependencies: '@jest/core': 29.7.0 '@jest/types': 29.6.3 import-local: 3.2.0 - jest-cli: 29.7.0(@types/node@18.19.55) + jest-cli: 29.7.0(@types/node@18.19.74) transitivePeerDependencies: - '@types/node' - babel-plugin-macros @@ -21028,7 +21047,7 @@ snapshots: lambda-cloud-node-api@1.0.1: {} - langchain@0.3.11(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(cheerio@1.0.0)(encoding@0.1.13)(handlebars@4.7.8)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)): + langchain@0.3.11(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(cheerio@1.0.0-rc.10)(encoding@0.1.13)(handlebars@4.7.8)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)): dependencies: '@langchain/core': 0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) '@langchain/openai': 0.3.17(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) @@ -21048,7 +21067,7 @@ snapshots: '@langchain/google-genai': 0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8) '@langchain/mistralai': 0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))) axios: 1.7.7 - cheerio: 1.0.0 + cheerio: 1.0.0-rc.10 handlebars: 4.7.8 transitivePeerDependencies: - encoding @@ -21649,11 +21668,16 @@ snapshots: napi-build-utils@1.0.2: {} - napi-build-utils@2.0.0: - optional: true - native-promise-only@0.8.1: {} + nats.ws@1.30.1: + optionalDependencies: + nkeys.js: 1.1.0 + + nats@2.29.1: + dependencies: + nkeys.js: 1.1.0 + natural-compare@1.4.0: {} ncp@2.0.0: @@ -21743,6 +21767,10 @@ snapshots: lolex: 5.1.2 path-to-regexp: 1.9.0 + nkeys.js@1.1.0: + dependencies: + tweetnacl: 1.0.3 + no-case@3.0.4: dependencies: lower-case: 2.0.2 @@ -21752,11 +21780,6 @@ snapshots: dependencies: semver: 7.6.3 - node-abi@3.73.0: - dependencies: - semver: 7.6.3 - optional: true - node-addon-api@6.1.0: {} node-addon-api@7.1.1: @@ -21806,7 +21829,7 @@ snapshots: node-mocks-http@1.16.0: dependencies: '@types/express': 4.17.21 - '@types/node': 18.19.50 + '@types/node': 18.19.74 accepts: 1.3.8 content-disposition: 0.5.4 depd: 1.1.2 @@ -21824,7 +21847,7 @@ snapshots: node-pty@1.0.0: dependencies: - nan: 2.17.0 + nan: 2.20.0 node-releases@2.0.18: {} @@ -22041,7 +22064,7 @@ snapshots: openai@4.78.1(encoding@0.1.13)(zod@3.23.8): dependencies: - '@types/node': 18.19.64 + '@types/node': 18.19.74 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -22211,11 +22234,6 @@ snapshots: domhandler: 5.0.3 parse5: 7.2.1 - parse5-parser-stream@7.1.2: - dependencies: - parse5: 7.2.1 - optional: true - parse5@6.0.1: {} parse5@7.2.1: @@ -22671,22 +22689,6 @@ snapshots: tar-fs: 2.1.1 tunnel-agent: 0.6.0 - prebuild-install@7.1.3: - dependencies: - detect-libc: 2.0.3 - expand-template: 2.0.3 - github-from-package: 0.0.0 - minimist: 1.2.8 - mkdirp-classic: 0.5.3 - napi-build-utils: 2.0.0 - node-abi: 3.73.0 - pump: 3.0.2 - rc: 1.2.8 - simple-get: 4.0.1 - tar-fs: 2.1.2 - tunnel-agent: 0.6.0 - optional: true - precond@0.2.3: {} predefine@0.1.3: @@ -22796,7 +22798,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 18.19.64 + '@types/node': 18.19.74 long: 5.2.3 protocol-buffers-schema@3.6.0: {} @@ -23384,7 +23386,7 @@ snapshots: react-property@2.0.0: {} - react-redux@8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@5.0.1): + react-redux@8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@4.2.1): dependencies: '@babel/runtime': 7.25.6 '@types/hoist-non-react-statics': 3.3.1 @@ -23397,7 +23399,7 @@ snapshots: '@types/react': 18.3.10 '@types/react-dom': 18.3.0 react-dom: 18.3.1(react@18.3.1) - redux: 5.0.1 + redux: 4.2.1 react-refresh@0.14.2: {} @@ -23517,9 +23519,6 @@ snapshots: dependencies: '@babel/runtime': 7.25.6 - redux@5.0.1: - optional: true - reflect-metadata@0.1.13: {} reflect.getprototypeof@1.0.6: @@ -23528,7 +23527,7 @@ snapshots: define-properties: 1.2.1 es-abstract: 1.23.3 es-errors: 1.3.0 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 globalthis: 1.0.4 which-builtin-type: 1.1.4 @@ -23769,8 +23768,8 @@ snapshots: safe-array-concat@1.1.2: dependencies: call-bind: 1.0.7 - get-intrinsic: 1.2.4 - has-symbols: 1.0.3 + get-intrinsic: 1.2.7 + has-symbols: 1.1.0 isarray: 2.0.5 safe-buffer@5.1.2: {} @@ -23933,8 +23932,8 @@ snapshots: define-data-property: 1.1.4 es-errors: 1.3.0 function-bind: 1.1.2 - get-intrinsic: 1.2.4 - gopd: 1.0.1 + get-intrinsic: 1.2.7 + gopd: 1.2.0 has-property-descriptors: 1.0.2 set-function-name@2.0.2: @@ -24068,7 +24067,7 @@ snapshots: dependencies: call-bind: 1.0.7 es-errors: 1.3.0 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 object-inspect: 1.13.2 side-channel@1.1.0: @@ -24320,7 +24319,7 @@ snapshots: es-abstract: 1.23.3 es-errors: 1.3.0 es-object-atoms: 1.0.0 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 gopd: 1.0.1 has-symbols: 1.0.3 internal-slot: 1.0.7 @@ -24338,19 +24337,19 @@ snapshots: call-bind: 1.0.7 define-properties: 1.2.1 es-abstract: 1.23.3 - es-object-atoms: 1.0.0 + es-object-atoms: 1.1.1 string.prototype.trimend@1.0.8: dependencies: call-bind: 1.0.7 define-properties: 1.2.1 - es-object-atoms: 1.0.0 + es-object-atoms: 1.1.1 string.prototype.trimstart@1.0.8: dependencies: call-bind: 1.0.7 define-properties: 1.2.1 - es-object-atoms: 1.0.0 + es-object-atoms: 1.1.1 string_decoder@0.10.31: {} @@ -24393,7 +24392,7 @@ snapshots: stripe@17.5.0: dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.74 qs: 6.14.0 strnum@1.0.5: {} @@ -24508,14 +24507,6 @@ snapshots: pump: 3.0.2 tar-stream: 2.2.0 - tar-fs@2.1.2: - dependencies: - chownr: 1.1.4 - mkdirp-classic: 0.5.3 - pump: 3.0.2 - tar-stream: 2.2.0 - optional: true - tar-fs@3.0.6: dependencies: pump: 3.0.2 @@ -24733,12 +24724,12 @@ snapshots: ts-dedent@2.2.0: {} - ts-jest@29.2.5(@babel/core@7.25.8)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.8))(jest@29.7.0(@types/node@18.19.55))(typescript@5.7.3): + ts-jest@29.2.5(@babel/core@7.25.8)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.8))(jest@29.7.0(@types/node@18.19.74))(typescript@5.7.3): dependencies: bs-logger: 0.2.6 ejs: 3.1.10 fast-json-stable-stringify: 2.1.0 - jest: 29.7.0(@types/node@18.19.55) + jest: 29.7.0(@types/node@18.19.74) jest-util: 29.7.0 json5: 2.2.3 lodash.memoize: 4.1.2 @@ -24796,6 +24787,8 @@ snapshots: dependencies: safe-buffer: 5.2.1 + tweetnacl@1.0.3: {} + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 @@ -24837,7 +24830,7 @@ snapshots: dependencies: call-bind: 1.0.7 for-each: 0.3.3 - gopd: 1.0.1 + gopd: 1.2.0 has-proto: 1.0.3 is-typed-array: 1.1.13 @@ -24846,7 +24839,7 @@ snapshots: available-typed-arrays: 1.0.7 call-bind: 1.0.7 for-each: 0.3.3 - gopd: 1.0.1 + gopd: 1.2.0 has-proto: 1.0.3 is-typed-array: 1.1.13 @@ -24854,7 +24847,7 @@ snapshots: dependencies: call-bind: 1.0.7 for-each: 0.3.3 - gopd: 1.0.1 + gopd: 1.2.0 has-proto: 1.0.3 is-typed-array: 1.1.13 possible-typed-array-names: 1.0.0 @@ -24902,18 +24895,13 @@ snapshots: dependencies: call-bind: 1.0.7 has-bigints: 1.0.2 - has-symbols: 1.0.3 + has-symbols: 1.1.0 which-boxed-primitive: 1.0.2 underscore@1.13.7: {} undici-types@5.26.5: {} - undici-types@6.20.0: {} - - undici@6.21.1: - optional: true - unicorn-magic@0.1.0: {} unified@10.1.2: @@ -25381,14 +25369,6 @@ snapshots: webworkify@1.5.0: {} - whatwg-encoding@3.1.1: - dependencies: - iconv-lite: 0.6.3 - optional: true - - whatwg-mimetype@4.0.0: - optional: true - whatwg-url@5.0.0: dependencies: tr46: 0.0.3 @@ -25431,7 +25411,7 @@ snapshots: available-typed-arrays: 1.0.7 call-bind: 1.0.7 for-each: 0.3.3 - gopd: 1.0.1 + gopd: 1.2.0 has-tostringtag: 1.0.2 which-typed-array@1.1.9: @@ -25439,7 +25419,7 @@ snapshots: available-typed-arrays: 1.0.5 call-bind: 1.0.7 for-each: 0.3.3 - gopd: 1.0.1 + gopd: 1.2.0 has-tostringtag: 1.0.0 is-typed-array: 1.1.10 From 916987e780fb42580f7b9d087d85bfcf672a2348 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 28 Jan 2025 18:17:25 +0000 Subject: [PATCH 054/281] nats: automate installation of nats, nsc and nats-server on a given host --- src/packages/backend/data.ts | 26 ++++-- src/packages/backend/nats/install.ts | 130 +++++++++++++++++++++++++++ src/packages/server/nats/auth.ts | 15 ++-- 3 files changed, 155 insertions(+), 16 deletions(-) create mode 100644 src/packages/backend/nats/install.ts diff --git a/src/packages/backend/data.ts b/src/packages/backend/data.ts index df8548e628..f9447fcbb1 100644 --- a/src/packages/backend/data.ts +++ b/src/packages/backend/data.ts @@ -47,7 +47,7 @@ interface CoCalcSSLEnvConfig extends Dict { SMC_DB_SSL_CA_FILE?: string; SMC_DB_SSL_CLIENT_CERT_FILE?: string; SMC_DB_SSL_CLIENT_KEY_FILE?: string; - SMC_DB_SSL_CLIENT_KEY_PASSPHRASE?:string; + SMC_DB_SSL_CLIENT_KEY_PASSPHRASE?: string; } // This interface is used to specify environment variables to be passed to the "psql" command for @@ -75,11 +75,14 @@ export interface PsqlSSLEnvConfig { // We extend the existing ConnectionOptions interface to include certificate file paths, since these // are used when connecting to Postgres outside of Node (e.g., for raw psql queries). // -export type SSLConfig = ConnectionOptions & { - caFile?: string; - clientCertFile?: string; - clientKeyFile?: string; -} | boolean | undefined; +export type SSLConfig = + | (ConnectionOptions & { + caFile?: string; + clientCertFile?: string; + clientKeyFile?: string; + }) + | boolean + | undefined; /** * Converts an environment-variable-driven SSLEnvConfig into a superset of the SSL context expected @@ -87,7 +90,9 @@ export type SSLConfig = ConnectionOptions & { * * @param env */ -export function sslConfigFromCoCalcEnv(env: CoCalcSSLEnvConfig = process.env): SSLConfig { +export function sslConfigFromCoCalcEnv( + env: CoCalcSSLEnvConfig = process.env, +): SSLConfig { const sslConfig: SSLConfig = {}; if (env.SMC_DB_SSL_CA_FILE) { @@ -101,7 +106,7 @@ export function sslConfigFromCoCalcEnv(env: CoCalcSSLEnvConfig = process.env): S } if (env.SMC_DB_SSL_CLIENT_KEY_FILE) { - sslConfig.clientKeyFile = env.SMC_DB_SSL_CLIENT_KEY_FILE + sslConfig.clientKeyFile = env.SMC_DB_SSL_CLIENT_KEY_FILE; sslConfig.key = readFileSync(env.SMC_DB_SSL_CLIENT_KEY_FILE); } @@ -109,7 +114,9 @@ export function sslConfigFromCoCalcEnv(env: CoCalcSSLEnvConfig = process.env): S sslConfig.passphrase = env.SMC_DB_SSL_CLIENT_KEY_PASSPHRASE; } - return isEmpty(sslConfig) ? (env.SMC_DB_SSL?.toLowerCase() === "true") : sslConfig; + return isEmpty(sslConfig) + ? env.SMC_DB_SSL?.toLowerCase() === "true" + : sslConfig; } /** @@ -174,6 +181,7 @@ export const secrets: string = process.env.SECRETS ?? join(data, "secrets"); export const logs: string = process.env.LOGS ?? join(data, "logs"); export const blobstore: "disk" | "sqlite" = (process.env.COCALC_JUPYTER_BLOBSTORE_IMPL as any) ?? "sqlite"; +export const nats: string = process.env.COCALC_NATS ?? join(data, "nats"); export let apiKey: string = process.env.API_KEY ?? ""; export let apiServer: string = process.env.API_SERVER ?? ""; diff --git a/src/packages/backend/nats/install.ts b/src/packages/backend/nats/install.ts new file mode 100644 index 0000000000..1ff1941c44 --- /dev/null +++ b/src/packages/backend/nats/install.ts @@ -0,0 +1,130 @@ +/* +Ensure installed specific correct versions of the following +three GO programs in {data}/nats/bin on this server, correct +for this architecture: + + - nats + - nats-server + - nsc + +We assume curl and python3 are installed. +*/ + +import { nats } from "@cocalc/backend/data"; +import { join } from "path"; +import getLogger from "@cocalc/backend/logger"; +import { pathExists } from "fs-extra"; +import { executeCode } from "@cocalc/backend/execute-code"; + +const VERSIONS = { + "nats-server": "v2.11.0-preview.2", + nats: "v0.1.6", +}; + +export const bin = join(nats, "bin"); +const logger = getLogger("backend:nats:install"); + +export async function install(noUpgrade = false) { + logger.debug("ensure nats binaries installed in ", bin); + + if (!(await pathExists(bin))) { + await executeCode({ command: "mkdir", args: ["-p", bin] }); + } + + await Promise.all([ + installNatsServer(noUpgrade), + installNsc(noUpgrade), + installNatsCli(noUpgrade), + ]); +} + +// call often, but runs at most once and ONLY does something if +// there is no binary i.e., it doesn't upgrade. +let installed = false; +export async function ensureInstalled() { + if (installed) { + return; + } + installed = true; + await install(true); +} + +async function getVersion(name: string) { + try { + const { stdout } = await executeCode({ + command: join(bin, name), + args: ["--version"], + }); + const v = stdout.trim().split(/\s/g); + return v[v.length - 1]; + } catch { + return ""; + } +} + +async function installNatsServer(noUpgrade) { + if (noUpgrade && (await pathExists(join(bin, "nats-server")))) { + return; + } + if ((await getVersion("nats-server")) == VERSIONS["nats-server"]) { + logger.debug( + `nats-server version ${VERSIONS["nats-server"]} already installed`, + ); + return; + } + logger.debug("installing nats-server"); + await executeCode({ + command: `curl -sf https://binaries.nats.dev/nats-io/nats-server/v2@${VERSIONS["nats-server"]} | sh`, + path: bin, + verbose: true, + }); +} + +export async function installNsc(noUpgrade) { + const nsc = join(bin, "nsc"); + if (noUpgrade && (await pathExists(nsc))) { + return; + } + + if (!(await pathExists(nsc))) { + await executeCode({ + command: `curl -LO https://raw.githubusercontent.com/nats-io/nsc/main/install.py`, + path: bin, + verbose: true, + }); + const { stdout } = await executeCode({ + path: bin, + env: { PYTHONDONTWRITEBYTECODE: 1 }, + command: + "python3 -c 'import os, sys; sys.path.insert(0,\".\"); import install; print(install.release_url(sys.platform, os.uname()[4], sys.argv[1] if len(sys.argv) > 1 else None))'", + }); + await executeCode({ + command: `curl -sL ${stdout.trim()} -o nsc.zip && unzip nsc.zip -d . && rm nsc.zip install.py`, + path: bin, + verbose: true, + }); + } else { + await executeCode({ + command: nsc, + args: ["update"], + path: bin, + verbose: true, + }); + } +} + +async function installNatsCli(noUpgrade) { + if (noUpgrade && (await pathExists(join(bin, "nats")))) { + return; + } + if ((await getVersion("nats")) == VERSIONS["nats"]) { + logger.debug(`nats version ${VERSIONS["nats"]} already installed`); + return; + } + logger.debug("installing nats cli"); + await executeCode({ + command: `curl -sf https://binaries.nats.dev/nats-io/natscli/nats@${VERSIONS["nats"]} | sh`, + path: bin, + verbose: true, + }); +} diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 3eb51b7c76..4162ed08c8 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -6,7 +6,7 @@ Points that took me a while to figure out: - a signing key, which allows access to hub api's and running projects they are a collaborator on, etc., etc. - a NATS user that has *bearer* enabled and is associated to the above signing key, so they get all its permissions - + Then the JWT for the user is stored as a secure http cookie in the browser and grants the user permissions. TODO: worry about expiration @@ -15,11 +15,10 @@ Points that took me a while to figure out: DOCS: - https://nats-io.github.io/nsc/ - + USAGE: -a = require('@cocalc/server/nats/auth') -await a.configureNatsUser({account_id:'275f1db7-bf37-4b44-b9aa-d64694269c9f'}) +a = require('@cocalc/server/nats/auth'); await a.configureNatsUser({account_id:'275f1db7-bf37-4b44-b9aa-d64694269c9f'}) await a.configureNatsUser({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}) */ @@ -27,6 +26,8 @@ import { executeCode } from "@cocalc/backend/execute-code"; import getPool from "@cocalc/database/pool"; import { isValidUUID } from "@cocalc/util/misc"; import getLogger from "@cocalc/backend/logger"; +import { bin, ensureInstalled } from "@cocalc/backend/nats/install"; +import { join } from "path"; import { throttle } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; @@ -39,11 +40,12 @@ export async function nsc( args: string[], { noAccount }: { noAccount?: boolean } = {}, ) { + await ensureInstalled(); // make sure (once) that nsc is installed // todo: for production we have to put some authentication // options, e.g., taken from the database. Skip that for now. // console.log(`nsc ${args.join(" ")}`); return await executeCode({ - command: "nsc", + command: join(bin, "nsc"), args: noAccount ? args : [...args, "-a", NATS_ACCOUNT], }); } @@ -120,9 +122,8 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { const goalSub = new Set(["_INBOX.>"]); if (userType == "account") { - goalSub.add(`$KV.account-${userId}.>`); - + const pool = getPool(); // all RUNNING projects with the user's group const query = `SELECT project_id, users#>>'{${userId},group}' AS group FROM projects WHERE state#>>'{state}'='running' AND users ? '${userId}' ORDER BY project_id`; From 534cb21e4f04c1b207c89ab01ab95f4e7a606fc8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 28 Jan 2025 21:49:00 +0000 Subject: [PATCH 055/281] nats: automate install/configuration/auth at least for dev mode --- src/package.json | 3 +- src/packages/backend/nats/conf.ts | 104 +++++++++++++++++++++++++++ src/packages/backend/nats/index.ts | 4 +- src/packages/backend/nats/install.ts | 2 +- src/packages/backend/nats/nsc.ts | 30 ++++++++ src/packages/backend/nats/server.ts | 15 ++++ src/packages/server/nats/auth.ts | 40 ++++------- src/scripts/g-tmux.sh | 15 ++-- 8 files changed, 177 insertions(+), 36 deletions(-) create mode 100644 src/packages/backend/nats/conf.ts create mode 100644 src/packages/backend/nats/nsc.ts create mode 100644 src/packages/backend/nats/server.ts diff --git a/src/package.json b/src/package.json index a855853d1b..e64adf3abc 100644 --- a/src/package.json +++ b/src/package.json @@ -18,7 +18,8 @@ "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test", "test": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r test", "prettier-all": "cd packages/", - "nats-server": "nats-server -c ./scripts/nats.conf" + "nats-server": "cd ${COCALC_ROOT:=$INIT_CWD}/data/nats && ./bin/nats-server -c server.conf", + "nats": "echo 'Starting NATS subshell where you can use: nsc, nats, nats-server'; XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:$PATH PS1=\"nats> \" bash" }, "repository": { "type": "git", diff --git a/src/packages/backend/nats/conf.ts b/src/packages/backend/nats/conf.ts new file mode 100644 index 0000000000..5254dd6749 --- /dev/null +++ b/src/packages/backend/nats/conf.ts @@ -0,0 +1,104 @@ +/* +Configure nats-server, i.e., generate configuration files. + + +echo "await require('@cocalc/backend/nats/conf').configureNatsServer()" | node + +*/ + +import { pathExists } from "fs-extra"; +import { data, nats } from "@cocalc/backend/data"; +import { join } from "path"; +import getLogger from "@cocalc/backend/logger"; +import { writeFile } from "fs/promises"; +import { NATS_JWT_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; +import nsc from "./nsc"; +import { executeCode } from "@cocalc/backend/execute-code"; +import { startServer } from "./server"; +import { kill } from "node:process"; + +const logger = getLogger("backend:nats:install"); + +// this is assumed in cocalc/src/package.json: +const confPath = join(nats, "server.conf"); + +// for now for local dev: +export const natsServerUrl = "nats://localhost:4222"; +export const natsAccountName = "cocalc"; + +export async function configureNatsServer() { + logger.debug("configureNatsServer", { confPath }); + if (await pathExists(confPath)) { + logger.debug( + `configureNatsServer: target conf file '${confPath}' already exists so not doing anything`, + ); + return; + } + + await writeFile( + confPath, + ` +jetstream: enabled + +jetstream { + store_dir: data/nats/jetstream +} + +websocket { + listen: "localhost:8443" + no_tls: true + jwt_cookie: "${NATS_JWT_COOKIE_NAME}" +} + +resolver { + type: full + dir: 'data/nats/jwt' + allow_delete: true + interval: "1m" + timeout: "3s" +} + +${await configureNsc()} +`, + ); + + const pid = startServer(); + // push initial operator/account/user configuration so its possible + // to configure other accounts + await nsc(["push", "-u", natsServerUrl]); + kill(pid); +} + +export async function configureNsc() { + // initialize the local nsc account config + await nsc(["init", "--name", natsAccountName]); + // set the url for the operat + await nsc(["edit", "operator", "--account-jwt-server-url", natsServerUrl]); + // make cocalc user able to pub and sub to everything + await nsc(["edit", "user", "--name", "cocalc", "--allow-pubsub", ">"]); + // enable jetstream for the cocalc account + await nsc(["edit", "account", "--js-mem-storage=-1", "--js-disk-storage=-1"]); + // set nats default context to cocalc user, so using the nats cli works. + await executeCode({ + command: join(nats, "bin", "nats"), + args: [ + "context", + "save", + "--select", + "--nsc=nsc://cocalc/cocalc/cocalc", + "cocalc", + ], + env: { + XDG_DATA_HOME: data, + XDG_CONFIG_HOME: data, + PATH: `${join(nats, "bin")}:${process.env.PATH}`, + }, + verbose: true, + }); + + // return the operator and system_account for inclusion in server config + const { stdout } = await nsc(["generate", "config", "--nats-resolver"]); + const i = stdout.indexOf("system_account"); + const j = stdout.indexOf("\n", i + 1); + return stdout.slice(0, j); +} diff --git a/src/packages/backend/nats/index.ts b/src/packages/backend/nats/index.ts index 622e3d486a..99539c4388 100644 --- a/src/packages/backend/nats/index.ts +++ b/src/packages/backend/nats/index.ts @@ -1,5 +1,5 @@ import { join } from "path"; -import { secrets } from "@cocalc/backend/data"; +import { nats } from "@cocalc/backend/data"; import { readFile } from "node:fs/promises"; import getLogger from "@cocalc/backend/logger"; import { connect, credsAuthenticator } from "nats"; @@ -7,7 +7,7 @@ import { connect, credsAuthenticator } from "nats"; const logger = getLogger("backend:nats"); export async function getCreds(): Promise { - const filename = join(secrets, "nats.creds"); + const filename = join(nats, "nsc/keys/creds/cocalc/cocalc/cocalc.creds"); try { return (await readFile(filename)).toString().trim(); } catch { diff --git a/src/packages/backend/nats/install.ts b/src/packages/backend/nats/install.ts index 1ff1941c44..2b7dfed270 100644 --- a/src/packages/backend/nats/install.ts +++ b/src/packages/backend/nats/install.ts @@ -12,9 +12,9 @@ We assume curl and python3 are installed. import { nats } from "@cocalc/backend/data"; import { join } from "path"; -import getLogger from "@cocalc/backend/logger"; import { pathExists } from "fs-extra"; import { executeCode } from "@cocalc/backend/execute-code"; +import getLogger from "@cocalc/backend/logger"; const VERSIONS = { "nats-server": "v2.11.0-preview.2", diff --git a/src/packages/backend/nats/nsc.ts b/src/packages/backend/nats/nsc.ts new file mode 100644 index 0000000000..29402c2aaf --- /dev/null +++ b/src/packages/backend/nats/nsc.ts @@ -0,0 +1,30 @@ +/* +Run the Nats nsc command line tool with appropriate environment. + +https://docs.nats.io/using-nats/nats-tools/nsc + +If you want to run nsc in a terminal, do this: + +# DATA=your data/ directory, with data/nats, etc. in it, e.g., +# in a dev install this is cocalc/src/data: + +export DATA=$HOME/cocalc/src/data +export PATH=$DATA/nats/bin:$PATH +export XDG_DATA_HOME=$DATA +export XDG_CONFIG_HOME=$DATA +*/ + +import { bin, ensureInstalled } from "./install"; +import { data } from "@cocalc/backend/data"; +import { join } from "path"; +import { executeCode } from "@cocalc/backend/execute-code"; + +export default async function nsc(args: string[]) { + await ensureInstalled(); // make sure (once) that nsc is installed + return await executeCode({ + command: join(bin, "nsc"), + args, + env: { XDG_DATA_HOME: data, XDG_CONFIG_HOME: data }, + verbose: true, + }); +} diff --git a/src/packages/backend/nats/server.ts b/src/packages/backend/nats/server.ts new file mode 100644 index 0000000000..7e8c3dedc0 --- /dev/null +++ b/src/packages/backend/nats/server.ts @@ -0,0 +1,15 @@ +import { nats } from "@cocalc/backend/data"; +import { join } from "path"; +import { spawn } from "node:child_process"; + +export function startServer(): number { + const { pid } = spawn( + join(nats, "bin", "nats-server"), + ["-c", join(nats, "server.conf")], + { cwd: nats }, + ); + if (pid == null) { + throw Error("issue spawning nats-server"); + } + return pid; +} diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 4162ed08c8..5af11d1262 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -22,32 +22,21 @@ a = require('@cocalc/server/nats/auth'); await a.configureNatsUser({account_id:' await a.configureNatsUser({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}) */ -import { executeCode } from "@cocalc/backend/execute-code"; import getPool from "@cocalc/database/pool"; import { isValidUUID } from "@cocalc/util/misc"; import getLogger from "@cocalc/backend/logger"; -import { bin, ensureInstalled } from "@cocalc/backend/nats/install"; -import { join } from "path"; +import nsc0 from "@cocalc/backend/nats/nsc"; +import { natsAccountName } from "@cocalc/backend/nats/conf"; import { throttle } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -// TODO: move this to server settings -const NATS_ACCOUNT = "cocalc"; - const logger = getLogger("server:nats:auth"); export async function nsc( args: string[], { noAccount }: { noAccount?: boolean } = {}, ) { - await ensureInstalled(); // make sure (once) that nsc is installed - // todo: for production we have to put some authentication - // options, e.g., taken from the database. Skip that for now. - // console.log(`nsc ${args.join(" ")}`); - return await executeCode({ - command: join(bin, "nsc"), - args: noAccount ? args : [...args, "-a", NATS_ACCOUNT], - }); + return await nsc0(noAccount ? args : [...args, "-a", natsAccountName]); } // TODO: consider making the names shorter strings using https://www.npmjs.com/package/short-uuid @@ -224,36 +213,31 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { } } -export async function getScopedSigningKey(natsUser: string) { +export async function getScopedSigningKey( + natsUser: string, +): Promise<{ [key: string]: string[] } | null> { let { stdout } = await nsc(["describe", "user", natsUser]); // it seems impossible to get the scoped signing key params using --json; they just aren't there - // i.e., it's not implemented. so we parse it. + // i.e., it's not implemented. so we parse text output... const i = stdout.indexOf("Scoped"); if (i == -1) { // there is no scoped signing key return null; } stdout = stdout.slice(i); - const obj: any = {}; + const obj: { [key: string]: string[] } = {}; let key = ""; for (const line of stdout.split("\n")) { const v = line.split("|"); if (v.length == 4) { const key2 = v[1].trim(); - let val: string | string[] = v[2].trim(); + const val = v[2].trim(); if (!key2 && obj[key] != null) { // automatically account for arrays (for pub and sub) - if (typeof obj[key] == "string") { - obj[key] = [obj[key], val]; - } else { - obj[key].push(val); - } + obj[key].push(val); } else { - key = key2; - if (key.startsWith("Pub ") || key.startsWith("Sub ")) { - val = [val]; - } - obj[key] = val; + key = key2; // record this so can use for arrays. Also, obj[key] is null since key2 is set. + obj[key] = [val]; } } } diff --git a/src/scripts/g-tmux.sh b/src/scripts/g-tmux.sh index 3ae4a2e3dd..179058ff2c 100755 --- a/src/scripts/g-tmux.sh +++ b/src/scripts/g-tmux.sh @@ -1,20 +1,27 @@ #!/usr/bin/env bash +echo "Spawning tmux windows with: hub, database, nats-server, rspack or memory monitor..." + export PWD=`pwd` tmux new-session -d -s mysession tmux new-window -t mysession:1 tmux new-window -t mysession:2 +tmux new-window -t mysession:3 +sleep 2 +tmux send-keys -t mysession:0 '$PWD/scripts/g.sh' C-m sleep 2 -tmux send-keys -t mysession:1 '$PWD/scripts/g.sh' C-m +tmux send-keys -t mysession:1 'pnpm database' C-m sleep 2 -tmux send-keys -t mysession:0 'pnpm database' C-m +tmux send-keys -t mysession:2 'pnpm nats-server' C-m if [ -n "$NO_RSPACK_DEV_SERVER" ]; then sleep 2 -tmux send-keys -t mysession:2 'pnpm rspack' C-m +tmux send-keys -t mysession:3 'pnpm rspack' C-m + else + sleep 2 -tmux send-keys -t mysession:2 '$PWD/scripts/memory_monitor.py' C-m +tmux send-keys -t mysession:3 '$PWD/scripts/memory_monitor.py' C-m fi tmux attach -t mysession:1 From 61ca26236ce290cc4e16c82745d53d71886701b3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 29 Jan 2025 00:26:19 +0000 Subject: [PATCH 056/281] nats: set variables to disable use of nats everywhere by default - this is so we can iteratively support things... --- .../frame-editors/terminal-editor/connected-terminal.ts | 4 +++- src/packages/sync/editor/generic/sync-doc.ts | 4 +++- src/packages/sync/table/synctable.ts | 6 +++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 11a3030152..1713303e27 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -11,6 +11,8 @@ extra support for being connected to: - frame-editor (via actions) */ +const USE_NATS = false; + import { callback, delay } from "awaiting"; import { Map } from "immutable"; import { debounce } from "lodash"; @@ -299,7 +301,7 @@ export class Terminal { }; async connect(): Promise { - if (this.path.startsWith("nats/")) { + if (USE_NATS && this.path.startsWith("nats/")) { return await this.connectNats(); } this.assert_not_closed(); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 8344e20254..f6499464d0 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -19,6 +19,8 @@ EVENTS: - ... TODO */ +const USE_NATS = false; + /* OFFLINE_THRESH_S - If the client becomes disconnected from the backend for more than this long then---on reconnect---do extra work to ensure that all snapshots are up to date (in @@ -275,7 +277,7 @@ export class SyncDoc extends EventEmitter { this[field] = opts[field]; } } - this.useNats = this.path.startsWith("nats/"); + this.useNats = USE_NATS && this.path.startsWith("nats/"); if (this.ephemeral) { // So the doctype written to the database reflects the // ephemeral state. Here ephemeral determines whether diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index f9af6f2807..22e6b610ac 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -17,6 +17,10 @@ ways of orchestrating a SyncTable. // info about every get/set let DEBUG: boolean = false; +// enable experimental nats database backed changefeed. +// for this to work you must explicitly run the server in @cocalc/database/nats +const USE_NATS = false; + export function set_debug(x: boolean): void { DEBUG = x; } @@ -730,7 +734,7 @@ export class SyncTable extends EventEmitter { let delay_ms: number = 500; while (true) { this.close_changefeed(); - if (this.client.is_browser() && !this.project_id) { + if (USE_NATS && this.client.is_browser() && !this.project_id) { this.changefeed = new NatsChangefeed({ client: this.client, query: this.query, From ab916b86a075b1b12748ce584a8fe24a6f104a06 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 29 Jan 2025 01:25:16 +0000 Subject: [PATCH 057/281] nats: use nats for project websocket request/reply api (totally automatic; everything except channels) --- src/packages/backend/execute-code.ts | 2 +- src/packages/frontend/client/nats.ts | 13 +++- .../frontend/project/websocket/api.ts | 27 +++++--- src/packages/project/browser-websocket/api.ts | 3 +- .../project/browser-websocket/server.ts | 4 ++ .../project/nats/browser-websocket-api.ts | 69 +++++++++++++++++++ src/packages/project/nats/index.ts | 8 +-- 7 files changed, 109 insertions(+), 17 deletions(-) create mode 100644 src/packages/project/nats/browser-websocket-api.ts diff --git a/src/packages/backend/execute-code.ts b/src/packages/backend/execute-code.ts index f1208eee1e..491d9143e3 100644 --- a/src/packages/backend/execute-code.ts +++ b/src/packages/backend/execute-code.ts @@ -149,7 +149,7 @@ async function executeCodeNoAggregate( opts.timeout ??= PROJECT_EXEC_DEFAULT_TIMEOUT_S; opts.ulimit_timeout ??= true; opts.err_on_exit ??= true; - opts.verbose ??= true; + opts.verbose ??= false; if (opts.verbose) { log.debug(`input: ${opts.command} ${opts.args?.join(" ")}`); diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index bdfe42cfe1..ecb93f5920 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -46,6 +46,15 @@ export class NatsClient { return this.nc; }); + projectWebsocketApi = async ({ project_id, mesg, timeout = 5000 }) => { + const nc = await this.getConnection(); + const subject = `project.${project_id}.browser-api`; + const resp = await nc.request(subject, this.jc.encode(mesg), { + timeout, + }); + return this.jc.decode(resp.data); + }; + private callHub = async ({ service = "api", name, @@ -55,9 +64,9 @@ export class NatsClient { name: string; args: any[]; }) => { - const c = await this.getConnection(); + const nc = await this.getConnection(); const subject = `hub.account.${this.client.account_id}.${service}`; - const resp = await c.request( + const resp = await nc.request( subject, this.jc.encode({ name, diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index ed9712c5bf..ad22d429cd 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -23,8 +23,9 @@ import type { import { syntax2tool } from "@cocalc/util/code-formatter"; import { DirectoryListingEntry } from "@cocalc/util/types"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { Channel, Mesg, NbconvertParams } from "@cocalc/comm/websocket/types"; +import type { Channel, Mesg, NbconvertParams } from "@cocalc/comm/websocket/types"; import call from "@cocalc/sync/client/call"; +import { webapp_client } from "@cocalc/frontend/webapp-client"; export class API { private conn; @@ -40,8 +41,16 @@ export class API { }); } - async call(mesg: Mesg, timeout_ms: number): Promise { - return await call(this.conn, mesg, timeout_ms); + async primusCall(mesg: Mesg, timeout: number): Promise { + return await call(this.conn, mesg, timeout); + } + + async call(mesg: Mesg, timeout: number): Promise { + return await webapp_client.nats_client.projectWebsocketApi({ + project_id: this.project_id, + mesg, + timeout, + }); } async version(): Promise { @@ -233,7 +242,7 @@ export class API { } async terminal(path: string, options: object = {}): Promise { - const channel_name = await this.call( + const channel_name = await this.primusCall( { cmd: "terminal", path: path, @@ -246,13 +255,13 @@ export class API { } async project_info(): Promise { - const channel_name = await this.call({ cmd: "project_info" }, 60000); + const channel_name = await this.primusCall({ cmd: "project_info" }, 60000); return this.conn.channel(channel_name); } // Get the lean *channel* for the given '.lean' path. async lean_channel(path: string): Promise { - const channel_name = await this.call( + const channel_name = await this.primusCall( { cmd: "lean_channel", path: path, @@ -264,7 +273,7 @@ export class API { // Get the x11 *channel* for the given '.x11' path. async x11_channel(path: string, display: number): Promise { - const channel_name = await this.call( + const channel_name = await this.primusCall( { cmd: "x11_channel", path, @@ -280,7 +289,7 @@ export class API { query: { [field: string]: any }, options: { [field: string]: any }[], ): Promise { - const channel_name = await this.call( + const channel_name = await this.primusCall( { cmd: "synctable_channel", query, @@ -349,7 +358,7 @@ export class API { // sync_channel, but obviously a more nuanced protocol // was required. async symmetric_channel(name: string): Promise { - const channel_name = await this.call( + const channel_name = await this.primusCall( { cmd: "symmetric_channel", name, diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts index bf45024e51..2b8fcfc433 100644 --- a/src/packages/project/browser-websocket/api.ts +++ b/src/packages/project/browser-websocket/api.ts @@ -54,6 +54,7 @@ const log = getLogger("websocket-api"); let primus: any = undefined; export function init_websocket_api(_primus: any): void { + primus = _primus; primus.on("connection", function (spark) { @@ -92,7 +93,7 @@ export function init_websocket_api(_primus: any): void { }); } -async function handleApiCall(data: Mesg, spark): Promise { +export async function handleApiCall(data: Mesg, spark): Promise { const client = getClient(); switch (data.cmd) { case "version": diff --git a/src/packages/project/browser-websocket/server.ts b/src/packages/project/browser-websocket/server.ts index ae1f1eb59c..81390ba4c6 100644 --- a/src/packages/project/browser-websocket/server.ts +++ b/src/packages/project/browser-websocket/server.ts @@ -12,6 +12,7 @@ import { Router } from "express"; import { Server } from "http"; import Primus from "primus"; import type { PrimusWithChannels } from "@cocalc/terminal"; +import { init as initNatsBrowserWebsocketApi } from "@cocalc/project/nats/browser-websocket-api"; // We are NOT using UglifyJS because it can easily take 3 blocking seconds of cpu // during project startup to save 100kb -- it just isn't worth it. Obviously, it @@ -56,5 +57,8 @@ export default function init(server: Server, basePath: string): Router { `waiting for clients to request primus.js (length=${library.length})...`, ); + // we also init the new nats server, which is meant to replace this: + initNatsBrowserWebsocketApi(); + return router; } diff --git a/src/packages/project/nats/browser-websocket-api.ts b/src/packages/project/nats/browser-websocket-api.ts new file mode 100644 index 0000000000..bf379ea1c5 --- /dev/null +++ b/src/packages/project/nats/browser-websocket-api.ts @@ -0,0 +1,69 @@ +/* +Implement the same protocol as browser-websocket was built on using primus, +but instead using NATS. + +How to do development (so in a dev project doing cc-in-cc dev): + +0. From the browser, send a terminate-handler message, so the handler running in the project stops: + + await cc.client.nats_client.projectWebsocketApi({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094', mesg:{cmd:"terminate"}}) + +1. Open a terminal in the project itself, which sets up the required environment variables, e.g., + - COCALC_NATS_JWT -- this has the valid JWT issued to grant the project rights to use nats + - COCALC_PROJECT_ID + +2. cd to your dev packages/project source code, e.g., ../cocalc/src/packages/project + +3. Do this: + + echo 'require("@cocalc/project/client").init(); require("@cocalc/project/nats/browser-websocket-api").init()' | DEBUG=cocalc:* DEBUG_CONSOLE=yes node + +4. Use the browser to see the project is on nats and works: + + await cc.client.nats_client.projectWebsocketApi({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094', mesg:{cmd:"listing"}}) + +5. In a terminal you can always tap into the message stream for a particular project (do `pnpm nats-shell` if necessary to setup your environment): + + nats sub --match-replies project.56eb622f-d398-489a-83ef-c09f1a1e8094.browser-api + +*/ + +import { getLogger } from "@cocalc/project/logger"; +import { JSONCodec } from "nats"; +import { project_id } from "@cocalc/project/data"; +import getConnection from "./connection"; +import { handleApiCall } from "@cocalc/project/browser-websocket/api"; + +const logger = getLogger("project:nats:browser-websocket-api"); + +const jc = JSONCodec(); + +export async function init() { + const nc = await getConnection(); + const subject = `project.${project_id}.browser-api`; + logger.debug(`initAPI -- NATS project subject '${subject}'`); + const sub = nc.subscribe(subject); + for await (const mesg of sub) { + const data = jc.decode(mesg.data) ?? ({} as any); + if (data.cmd == "terminate") { + logger.debug( + "received terminate-handler, so will not handle any further messages", + ); + mesg.respond(jc.encode({ exiting: true })); + return; + } + handleRequest(data, mesg); + } +} + +async function handleRequest(data, mesg) { + let resp; + logger.debug("received cmd:", data?.cmd); + try { + resp = await handleApiCall(data, {}); + } catch (err) { + resp = { error: `${err}` }; + } + //logger.debug("responded", resp); + mesg.respond(jc.encode(resp)); +} diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts index 3de6213f95..e740476d58 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/nats/index.ts @@ -4,17 +4,17 @@ How to do development (so in a dev project doing cc-in-cc dev). 1. Open a terminal in the project itself, which sets up the required environment variables, e.g., - COCALC_NATS_JWT -- this has the valid JWT issued to grant the project rights to use nats - COCALC_PROJECT_ID - + 2. cd to your dev packages/project source code, e.g., ../cocalc/src/packages/project 3. Do this: - + echo 'require("@cocalc/project/nats").default()' | DEBUG=cocalc:* DEBUG_CONSOLE=yes node - + 4. Use the browser to see the project is on nats and works: await cc.client.nats_client.project({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e', endpoint:"exec", params:{command:'echo $COCALC_PROJECT_ID'}}) - + */ import { getLogger } from "@cocalc/project/logger"; From 1c50f13585f6d1eecf1d9ee5631adec27c99f626 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 29 Jan 2025 02:47:29 +0000 Subject: [PATCH 058/281] nats: automate adding permissions when connecting to project --- .../frontend/project/websocket/api.ts | 181 ++++++++++-------- src/packages/hub/hub.ts | 14 +- src/packages/nats/api/system.ts | 4 +- src/packages/server/nats/api/index.ts | 49 +++-- src/packages/server/nats/api/system.ts | 2 + src/packages/server/nats/auth.ts | 40 +++- src/packages/server/nats/index.ts | 1 + 7 files changed, 184 insertions(+), 107 deletions(-) diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index ad22d429cd..c8fa867d31 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -23,7 +23,11 @@ import type { import { syntax2tool } from "@cocalc/util/code-formatter"; import { DirectoryListingEntry } from "@cocalc/util/types"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import type { Channel, Mesg, NbconvertParams } from "@cocalc/comm/websocket/types"; +import type { + Channel, + Mesg, + NbconvertParams, +} from "@cocalc/comm/websocket/types"; import call from "@cocalc/sync/client/call"; import { webapp_client } from "@cocalc/frontend/webapp-client"; @@ -41,19 +45,35 @@ export class API { }); } - async primusCall(mesg: Mesg, timeout: number): Promise { + private primusCall = async (mesg: Mesg, timeout: number) => { return await call(this.conn, mesg, timeout); - } + }; - async call(mesg: Mesg, timeout: number): Promise { + private _call = async (mesg: Mesg, timeout: number): Promise => { return await webapp_client.nats_client.projectWebsocketApi({ project_id: this.project_id, mesg, timeout, }); - } + }; - async version(): Promise { + call = async (mesg: Mesg, timeout: number) => { + try { + return await this._call(mesg, timeout); + } catch (err) { + if (err.code == "PERMISSIONS_VIOLATION") { + // request update of our credentials to include this project, then try again + await webapp_client.nats_client.hub.system.addProjectPermission({ + project_id: this.project_id, + }); + return await this._call(mesg, timeout); + } else { + throw err; + } + } + }; + + version = async (): Promise => { // version can never change (except when you restart the project!), so its safe to cache if (this.cachedVersion != null) { return this.cachedVersion; @@ -71,84 +91,86 @@ export class API { return 0; } return this.cachedVersion; - } + }; - async delete_files( + delete_files = async ( paths: string[], compute_server_id?: number, - ): Promise { + ): Promise => { return await this.call( { cmd: "delete_files", paths, compute_server_id }, 60000, ); - } + }; // Move the given paths to the dest. The folder dest must exist // already and be a directory, or this is in an error. - async move_files( + move_files = async ( paths: string[], dest: string, compute_server_id?: number, - ): Promise { + ): Promise => { return await this.call( { cmd: "move_files", paths, dest, compute_server_id }, 60000, ); - } + }; // Rename the file src to be the file dest. The dest may be // in a different directory or may even exist already (in which) // case it is overwritten if it is a file. If dest exists and // is a directory, it is an error. - async rename_file( + rename_file = async ( src: string, dest: string, compute_server_id?: number, - ): Promise { + ): Promise => { return await this.call( { cmd: "rename_file", src, dest, compute_server_id }, 30000, ); - } + }; - async listing( + listing = async ( path: string, hidden: boolean = false, timeout: number = 15000, compute_server_id: number = 0, - ): Promise { + ): Promise => { return await this.call( { cmd: "listing", path, hidden, compute_server_id }, timeout, ); - } + }; /* Normalize the given paths relative to the HOME directory. This takes any old weird looking mess of a path and makes it one that can be opened properly with our file editor, and the path appears to be to a file *in* the HOME directory. */ - async canonical_path(path: string): Promise { + canonical_path = async (path: string): Promise => { const v = await this.canonical_paths([path]); const x = v[0]; if (typeof x != "string") { throw Error("bug in canonical_path"); } return x; - } - async canonical_paths(paths: string[]): Promise { + }; + canonical_paths = async (paths: string[]): Promise => { return await this.call({ cmd: "canonical_paths", paths }, 15000); - } + }; - async configuration( + configuration = async ( aspect: ConfigurationAspect, no_cache = false, - ): Promise { + ): Promise => { return await this.call({ cmd: "configuration", aspect, no_cache }, 15000); - } + }; // use the returned FormatterOptions for the API formatting call! - private check_formatter_available(config: FormatterConfig): FormatterOptions { + private check_formatter_available = ( + config: FormatterConfig, + ): FormatterOptions => { const formatting = this.get_formatting(); if (formatting == null) { throw new Error( @@ -166,9 +188,9 @@ export class API { ); } return { parser: tool }; - } + }; - get_formatting(): Capabilities | undefined { + get_formatting = (): Capabilities | undefined => { const project_store = redux.getProjectStore(this.project_id) as any; const configuration = project_store.get( "configuration", @@ -183,24 +205,24 @@ export class API { } else { return {} as Capabilities; } - } + }; // Returns { status: "ok", patch:... the patch} or // { status: "error", phase: "format", error: err.message }. // We return a patch rather than the entire file, since often // the file is very large, but the formatting is tiny. This is purely // a data compression technique. - async formatter(path: string, config: FormatterConfig): Promise { + formatter = async (path: string, config: FormatterConfig): Promise => { const options: FormatterOptions = this.check_formatter_available(config); // TODO change this to "formatter" at some point in the future (Sep 2020) return await this.call({ cmd: "prettier", path: path, options }, 15000); - } + }; - async formatter_string( + formatter_string = async ( str: string, config: FormatterConfig, timeout_ms: number = 15000, - ): Promise { + ): Promise => { const options: FormatterOptions = this.check_formatter_available(config); // TODO change this to "formatter_string" at some point in the future (Sep 2020) return await this.call( @@ -211,37 +233,40 @@ export class API { }, timeout_ms, ); - } + }; - async jupyter( + jupyter = async ( path: string, endpoint: string, query: any = undefined, timeout_ms: number = 20000, - ): Promise { + ): Promise => { return await this.call( { cmd: "jupyter", path, endpoint, query }, timeout_ms, ); - } + }; - async exec(opts: any): Promise { + exec = async (opts: any): Promise => { let timeout_ms = 10000; if (opts.timeout) { timeout_ms = opts.timeout * 1000 + 2000; } return await this.call({ cmd: "exec", opts }, timeout_ms); - } + }; - async eval_code(code: string, timeout_ms: number = 20000): Promise { + eval_code = async ( + code: string, + timeout_ms: number = 20000, + ): Promise => { return await this.call({ cmd: "eval_code", code }, timeout_ms); - } + }; - async realpath(path: string): Promise { + realpath = async (path: string): Promise => { return await this.call({ cmd: "realpath", path }, 15000); - } + }; - async terminal(path: string, options: object = {}): Promise { + terminal = async (path: string, options: object = {}): Promise => { const channel_name = await this.primusCall( { cmd: "terminal", @@ -252,15 +277,15 @@ export class API { ); //console.log(path, "got terminal channel", channel_name); return this.conn.channel(channel_name); - } + }; - async project_info(): Promise { + project_info = async (): Promise => { const channel_name = await this.primusCall({ cmd: "project_info" }, 60000); return this.conn.channel(channel_name); - } + }; // Get the lean *channel* for the given '.lean' path. - async lean_channel(path: string): Promise { + lean_channel = async (path: string): Promise => { const channel_name = await this.primusCall( { cmd: "lean_channel", @@ -269,10 +294,10 @@ export class API { 60000, ); return this.conn.channel(channel_name); - } + }; // Get the x11 *channel* for the given '.x11' path. - async x11_channel(path: string, display: number): Promise { + x11_channel = async (path: string, display: number): Promise => { const channel_name = await this.primusCall( { cmd: "x11_channel", @@ -282,13 +307,13 @@ export class API { 60000, ); return this.conn.channel(channel_name); - } + }; // Get the sync *channel* for the given SyncTable project query. - async synctable_channel( + synctable_channel = async ( query: { [field: string]: any }, options: { [field: string]: any }[], - ): Promise { + ): Promise => { const channel_name = await this.primusCall( { cmd: "synctable_channel", @@ -299,51 +324,51 @@ export class API { ); // console.log("synctable_channel", query, options, channel_name); return this.conn.channel(channel_name); - } + }; // Command-response API for synctables. // - mesg = {cmd:'close'} -- closes the synctable, even if persistent. - async syncdoc_call( + syncdoc_call = async ( path: string, mesg: { [field: string]: any }, timeout_ms: number = 30000, // ms timeout for call - ): Promise { + ): Promise => { return await this.call({ cmd: "syncdoc_call", path, mesg }, timeout_ms); - } + }; // Do a request/response command to the lean server. - async lean(opts: any): Promise { + lean = async (opts: any): Promise => { let timeout_ms = 10000; if (opts.timeout) { timeout_ms = opts.timeout * 1000 + 2000; } return await this.call({ cmd: "lean", opts }, timeout_ms); - } + }; // Convert a notebook to some other format. // --to options are listed in packages/frontend/jupyter/nbconvert.tsx // and implemented in packages/project/jupyter/convert/index.ts - async jupyter_nbconvert(opts: NbconvertParams): Promise { + jupyter_nbconvert = async (opts: NbconvertParams): Promise => { return await this.call( { cmd: "jupyter_nbconvert", opts }, (opts.timeout ?? 60) * 1000 + 5000, ); - } + }; // Get contents of an ipynb file, but with output and attachments removed (to save space) - async jupyter_strip_notebook(ipynb_path: string): Promise { + jupyter_strip_notebook = async (ipynb_path: string): Promise => { return await this.call( { cmd: "jupyter_strip_notebook", ipynb_path }, 15000, ); - } + }; // Run the notebook filling in the output of all cells, then return the // result as a string. Note that the output size (per cell and total) // and run time is bounded to avoid the output being HUGE, even if the // input is dumb. - async jupyter_run_notebook(opts: RunNotebookOptions): Promise { + jupyter_run_notebook = async (opts: RunNotebookOptions): Promise => { const max_total_time_ms = opts.limits?.max_total_time_ms ?? 20 * 60 * 1000; return await this.call( { cmd: "jupyter_run_notebook", opts }, @@ -352,12 +377,12 @@ export class API { // timer do the job, than have to wait for this generic timeout here, // since we want to at least get output for problems that ran. ); - } + }; // I think this isn't used. It was going to support // sync_channel, but obviously a more nuanced protocol // was required. - async symmetric_channel(name: string): Promise { + symmetric_channel = async (name: string): Promise => { const channel_name = await this.primusCall( { cmd: "symmetric_channel", @@ -366,22 +391,22 @@ export class API { 30000, ); return this.conn.channel(channel_name); - } + }; // Do a database query, but via the project. This has the project // do the query, so the identity used to access the database is that // of the project. This isn't useful in the browser, where the user // always has more power to directly use the database. It is *is* // very useful when using a project-specific api key. - async query(opts: any): Promise { + query = async (opts: any): Promise => { if (opts.timeout == null) { opts.timeout = 30; } const timeout_ms = opts.timeout * 1000 + 2000; return await this.call({ cmd: "query", opts }, timeout_ms); - } + }; - async computeServerSyncRequest(compute_server_id: number) { + computeServerSyncRequest = async (compute_server_id: number) => { if (!(typeof compute_server_id == "number" && compute_server_id > 0)) { throw Error("compute_server_id must be a positive integer"); } @@ -392,29 +417,29 @@ export class API { }, 30000, ); - } + }; - async copyFromProjectToComputeServer(opts: { + copyFromProjectToComputeServer = async (opts: { compute_server_id: number; paths: string[]; dest?: string; timeout?: number; - }) { + }) => { await this.call( { cmd: "copy_from_project_to_compute_server", opts }, (opts.timeout ?? 60) * 1000, ); - } + }; - async copyFromComputeServerToProject(opts: { + copyFromComputeServerToProject = async (opts: { compute_server_id: number; paths: string[]; dest?: string; timeout?: number; - }) { + }) => { await this.call( { cmd: "copy_from_compute_server_to_project", opts }, (opts.timeout ?? 60) * 1000, ); - } + }; } diff --git a/src/packages/hub/hub.ts b/src/packages/hub/hub.ts index e36585b2ff..3fccbf9bfa 100644 --- a/src/packages/hub/hub.ts +++ b/src/packages/hub/hub.ts @@ -41,7 +41,7 @@ import { start as startHubRegister } from "./hub_register"; import { getLogger } from "./logger"; import initDatabase, { database } from "./servers/database"; import initExpressApp from "./servers/express-app"; -//import initNatsServer from "@cocalc/server/nats"; +import initNatsServer from "@cocalc/server/nats"; import initHttpRedirect from "./servers/http-redirect"; import initPrimus from "./servers/primus"; import initVersionServer from "./servers/version"; @@ -179,6 +179,10 @@ async function startServer(): Promise { initIdleTimeout(projectControl); } + if (program.natsServer) { + await initNatsServer(); + } + if (program.websocketServer) { // Initialize the version server -- must happen after updating schema // (for first ever run). @@ -400,11 +404,12 @@ async function main(): Promise { "--all", "runs all of the servers: websocket, proxy, next (so you don't have to pass all those opts separately), and also mentions updator and updates db schema on startup; use this in situations where there is a single hub that serves everything (instead of a microservice situation like kucalc)", ) - .option("--websocket-server", "run the websocket server") - .option("--proxy-server", "run the proxy server") + .option("--websocket-server", "run a websocket server in this process") + .option("--nats-server", "run a hub nats API server in this process") + .option("--proxy-server", "run a proxy server in this process") .option( "--next-server", - "run the nextjs server (landing pages, share server, etc.)", + "run a nextjs server (landing pages, share server, etc.) in this process", ) .option( "--https-key [string]", @@ -499,6 +504,7 @@ async function main(): Promise { } if (program.all) { program.websocketServer = + program.natsServer = program.proxyServer = program.nextServer = program.mentions = diff --git a/src/packages/nats/api/system.ts b/src/packages/nats/api/system.ts index 26e5449610..deac71fc47 100644 --- a/src/packages/nats/api/system.ts +++ b/src/packages/nats/api/system.ts @@ -1,12 +1,14 @@ -import { noAuth } from "./util"; +import { noAuth, authFirst } from "./util"; import type { Customize } from "@cocalc/util/db-schema/server-settings"; export const system = { getCustomize: noAuth, ping: noAuth, + addProjectPermission: authFirst, }; export interface System { getCustomize: (fields?: string[]) => Promise; ping: () => { now: number }; + addProjectPermission: (opts: { project_id: string }) => Promise; } diff --git a/src/packages/server/nats/api/index.ts b/src/packages/server/nats/api/index.ts index 4708e8d5eb..47bc85a31c 100644 --- a/src/packages/server/nats/api/index.ts +++ b/src/packages/server/nats/api/index.ts @@ -1,33 +1,44 @@ -/* +/* This is meant to be similar to the nexts pages http api/v2, but using NATS instead of HTTPS. -To do development turn off nats-server handling for the hub, and run this script standalone: +To do development: + +1. turn off nats-server handling for the hub by sending this message from a browser as an admin: + + await cc.client.nats_client.callHub({name:"terminate"}) + +2. Run this script standalone: echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node - -Optional: start more servers -- requests get randomly routed to exactly one of them: + +3. Optional: start more servers -- requests get randomly routed to exactly one of them: echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node - - + + To make use of this from a browser: - await cc.client.nats_client.api({name:"customize", args:{fields:['siteName']}}) + await cc.client.nats_client.hub.system.getCustomize(['siteName']) + +or + + await cc.client.nats_client.callHub({name:"system.getCustomize", args:[['siteName']]}) -When you make changes, just restart the above. All clients will instantly -use the new version after you restart, and there is no need to restart the hub +When you make changes, just restart the above. All clients will instantly +use the new version after you restart, and there is no need to restart the hub itself or any clients. To view all requests (and replies) in realtime: - nats sub api.v2 --match-replies + nats sub api.v2 --match-replies */ import { JSONCodec } from "nats"; import getLogger from "@cocalc/backend/logger"; import { type HubApi, getUserId, transformArgs } from "@cocalc/nats/api/index"; import { getConnection } from "@cocalc/backend/nats"; +import userIsInGroup from "@cocalc/server/accounts/is-in-group"; const logger = getLogger("server:nats:api"); @@ -41,23 +52,31 @@ export async function initAPI() { const nc = await getConnection(); const sub = nc.subscribe(subject, { queue: "0" }); for await (const mesg of sub) { - handleApiRequest(mesg); + const request = jc.decode(mesg.data) ?? ({} as any); + if (request.name == "terminate") { + // special hook so admin can terminate handling. This is useful for development. + const { account_id } = getUserId(mesg.subject); + if (!(!!account_id && (await userIsInGroup(account_id, "admin")))) { + throw Error("only admin can terminate"); + } + mesg.respond(jc.encode({ status: "terminating" })); + return; + } + handleApiRequest(request, mesg); } } -async function handleApiRequest(mesg) { +async function handleApiRequest(request, mesg) { let resp; try { const { account_id, project_id } = getUserId(mesg.subject); - const request = jc.decode(mesg.data) ?? {}; const { name, args } = request as any; logger.debug("handling hub.api request:", { account_id, project_id, name, - args, }); - resp = await getResponse({ name, args, account_id, project_id }); + resp = (await getResponse({ name, args, account_id, project_id })) ?? null; } catch (err) { resp = { error: `${err}` }; } diff --git a/src/packages/server/nats/api/system.ts b/src/packages/server/nats/api/system.ts index 770441478c..c964eb5b6a 100644 --- a/src/packages/server/nats/api/system.ts +++ b/src/packages/server/nats/api/system.ts @@ -1,5 +1,7 @@ import getCustomize from "@cocalc/database/settings/customize"; export { getCustomize }; +import { addProjectPermission } from "@cocalc/server/nats/auth"; +export { addProjectPermission }; export function ping() { return { now: Date.now() }; diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 5af11d1262..d67a47a404 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -29,6 +29,7 @@ import nsc0 from "@cocalc/backend/nats/nsc"; import { natsAccountName } from "@cocalc/backend/nats/conf"; import { throttle } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import isCollaborator from "@cocalc/server/projects/is-collaborator"; const logger = getLogger("server:nats:auth"); @@ -118,13 +119,11 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { const query = `SELECT project_id, users#>>'{${userId},group}' AS group FROM projects WHERE state#>>'{state}'='running' AND users ? '${userId}' ORDER BY project_id`; const { rows } = await pool.query(query); for (const { project_id /*, group */ } of rows) { - // TODO - unsure -- do we need proven identity *in* project? - //goalPub.add(`project.${project_id}.api.${group}.${userId}`); goalPub.add(`project.${project_id}.>`); goalSub.add(`project.${project_id}.>`); - goalPub.add(`$KV.project-${project_id}.>`); - goalSub.add(`$KV.project-${project_id}.>`); + goalPub.add(`*.project-${project_id}.>`); + goalSub.add(`*.project-${project_id}.>`); } // TODO: there will be other subjects // TODO: something similar for projects, e.g., they can publish to a channel that browser clients @@ -134,12 +133,9 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { goalPub.add(`project.${userId}.>`); goalSub.add(`project.${userId}.>`); - goalPub.add(`$KV.project-${userId}.>`); - goalSub.add(`$KV.project-${userId}.>`); + goalPub.add(`*.project-${userId}.>`); + goalSub.add(`*.project-${userId}.>`); } - // TEMPORARY: for learning jetstream! - goalPub.add("$JS.>"); - goalSub.add("$JS.>"); // **Subject Permissions SYNC Algorithm ** // figure out what signing key currently allows an update it to be exactly what is specified above. @@ -213,6 +209,24 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { } } +export async function addProjectPermission({ account_id, project_id }) { + if (!(await isCollaborator({ account_id, project_id }))) { + throw Error("user must be collaborator on project"); + } + const name = getNatsUserName({ account_id }); + await nsc([ + "edit", + "signing-key", + "--sk", + name, + "--allow-sub", + `project.${project_id}.>,*.project.${project_id}.>`, + "--allow-pub", + `project.${project_id}.>,*.project.${project_id}.>`, + ]); + await pushToServer(); +} + export async function getScopedSigningKey( natsUser: string, ): Promise<{ [key: string]: string[] } | null> { @@ -277,6 +291,14 @@ export async function createNatsUser(cocalcUser: CoCalcUser) { pushToServer(); } +// export async function updateActiveCollaborators(project_id: string) { +// const pool = getPool(); +// const { rows } = await pool.query( +// "select account_id from accounts where account_id=any(select jsonb_object_keys(users)::uuid from projects where project_id=$1) and last_active >= now() - interval '1 day'", +// [project_id], +// ); +// } + export async function getJwt(cocalcUser: CoCalcUser): Promise { try { return await getNatsUserJwt(cocalcUser); diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts index 2ccc38944c..d4603354b5 100644 --- a/src/packages/server/nats/index.ts +++ b/src/packages/server/nats/index.ts @@ -5,5 +5,6 @@ const logger = getLogger("server:nats"); export default async function initNatsServer() { logger.debug("initializing nats cocalc hub server"); + // do NOT await this! initAPI(); } From d920317775d9ff2c04ca83a799793497fc3e3483 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 29 Jan 2025 03:05:36 +0000 Subject: [PATCH 059/281] nats: shorter ping times for now so nats clients start working again quickly after server restart or network outage --- src/packages/backend/nats/index.ts | 2 ++ src/packages/frontend/client/nats.ts | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/packages/backend/nats/index.ts b/src/packages/backend/nats/index.ts index 99539c4388..c7558f0d87 100644 --- a/src/packages/backend/nats/index.ts +++ b/src/packages/backend/nats/index.ts @@ -26,6 +26,8 @@ export async function getConnection() { const creds = await getCreds(); nc = await connect({ authenticator: credsAuthenticator(new TextEncoder().encode(creds)), + // bound on how long after network or server goes down until starts working again + pingInterval: 10000, }); logger.debug(`connected to ${nc.getServer()}`); } diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index ecb93f5920..677eb6efea 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -34,6 +34,9 @@ export class NatsClient { try { this.nc = await nats.connect({ servers: [server], + // this pingInterval determines how long from when the browser's network connection dies + // and comes back, until nats starts working again. + pingInterval: 10000, }); } catch (err) { console.log("set the JWT cookie and try again"); From 44631051771affe2358d2fbcc420caf565b8ac14 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 29 Jan 2025 05:03:09 +0000 Subject: [PATCH 060/281] nats: working on implementing sockets/channels compat layer (not done) --- src/packages/frontend/client/nats.ts | 17 +++++-- .../terminal-editor/connected-terminal.ts | 5 +- .../frontend/project/websocket/api.ts | 12 +++++ src/packages/nats/package.json | 3 +- src/packages/nats/socket.ts | 47 +++++++++++++++++++ src/packages/nats/util.ts | 7 +++ src/packages/terminal/lib/terminal.ts | 9 ++-- src/packages/terminal/lib/types.ts | 1 - 8 files changed, 89 insertions(+), 12 deletions(-) create mode 100644 src/packages/nats/socket.ts create mode 100644 src/packages/nats/util.ts diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 677eb6efea..c701cfa20b 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -6,10 +6,12 @@ import { join } from "path"; import { redux } from "../app-framework"; import * as jetstream from "@nats-io/jetstream"; import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; +import { randomId } from "@cocalc/nats/util"; import { parse_query } from "@cocalc/sync/table/util"; import { sha1 } from "@cocalc/util/misc"; import { keys } from "lodash"; import { type HubApi, initHubApi } from "@cocalc/nats/api/index"; +import { Socket } from "@cocalc/nats/socket"; export class NatsClient { /*private*/ client: WebappClient; @@ -19,6 +21,7 @@ export class NatsClient { public nats = nats; public jetstream = jetstream; public hub: HubApi; + public sessionId = randomId(); constructor(client: WebappClient) { this.client = client; @@ -30,7 +33,7 @@ export class NatsClient { return this.nc; } const server = `${location.protocol == "https:" ? "wss" : "ws"}://${location.host}${appBasePath}/nats`; - console.log(`connecting to ${server}...`); + console.log(`NATS: connecting to ${server}...`); try { this.nc = await nats.connect({ servers: [server], @@ -39,13 +42,13 @@ export class NatsClient { pingInterval: 10000, }); } catch (err) { - console.log("set the JWT cookie and try again"); + console.log("NATS: set the JWT cookie and try again"); await fetch(join(appBasePath, "nats")); this.nc = await nats.connect({ servers: [server], }); } - console.log(`connected to ${server}`); + console.log(`NATS: connected to ${server}`); return this.nc; }); @@ -179,4 +182,12 @@ export class NatsClient { this.changefeedInterest(query, true); return await this.synctable(query, { atomic: true }); }; + + createSocket = async (subjects: { listen: string; send: string }) => { + return new Socket({ + ...subjects, + nc: await this.getConnection(), + jc: this.jc, + }); + }; } diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 1713303e27..ba96ded7c3 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -533,6 +533,7 @@ export class Terminal { rows?: number; cols?: number; payload: any; + ignore?: string; id?: number; }): void { //console.log("handle_mesg", this.id, mesg); @@ -555,7 +556,9 @@ export class Terminal { this.no_ignore(); break; case "close": - this.close_request(); + if (mesg.ignore != this.id) { + this.close_request(); + } break; case "computeServerId": if (this.actions.store != null && this.actions.setState != null) { diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index c8fa867d31..ccc1eb3a11 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -279,6 +279,18 @@ export class API { return this.conn.channel(channel_name); }; + terminal1 = async (path: string, options: object = {}): Promise => { + const subjects = await this.call( + { + cmd: "terminal", + path, + options, + }, + 20000, + ); + return (await webapp_client.nats_client.createSocket(subjects)) as any; + }; + project_info = async (): Promise => { const channel_name = await this.primusCall({ cmd: "project_info" }, 60000); return this.conn.channel(channel_name); diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 778d8f24c5..5e2877fb60 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -5,7 +5,8 @@ "exports": { "./sync/*": "./dist/sync/*.js", "./api": "./dist/api/index.js", - "./api/*": "./dist/api/*.js" + "./api/*": "./dist/api/*.js", + "./*": "./dist/*.js" }, "scripts": { "preinstall": "npx only-allow pnpm", diff --git a/src/packages/nats/socket.ts b/src/packages/nats/socket.ts new file mode 100644 index 0000000000..648e2d01b5 --- /dev/null +++ b/src/packages/nats/socket.ts @@ -0,0 +1,47 @@ +/* +Implement a websocket as exposed in Primus over NATS. +*/ + +import { EventEmitter } from "events"; + +export class Socket extends EventEmitter { + private listen: string; + private send: string; + private nc; + private jc; + + constructor({ + listen, + send, + nc, + jc, + }: { + // subject to listen on + listen: string; + // subject to write to + send: string; + // nats connection + nc; + // json codec + jc; + }) { + super(); + this.listen = listen; + this.send = send; + this.nc = nc; + this.jc = jc; + this.startListening(); + } + + private startListening = async () => { + const sub = this.nc.subscribe(this.listen); + for await (const mesg of sub) { + const { data } = this.jc.decode(mesg.data) ?? ({} as any); + this.emit("data", data); + } + }; + + write(data) { + this.nc.publish(this.send, this.jc.encode({ data })); + } +} diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts new file mode 100644 index 0000000000..a9f827acd6 --- /dev/null +++ b/src/packages/nats/util.ts @@ -0,0 +1,7 @@ +import generateVouchers from "@cocalc/util/vouchers"; + +// nice alphanumeric string that can be used as nats subject, and very +// unlikely to randomly collide with another browser tab from this account. +export function randomId() { + return generateVouchers({ count: 1, length: 10 })[0]; +} diff --git a/src/packages/terminal/lib/terminal.ts b/src/packages/terminal/lib/terminal.ts index 6e66b99964..0195fd6501 100644 --- a/src/packages/terminal/lib/terminal.ts +++ b/src/packages/terminal/lib/terminal.ts @@ -506,12 +506,9 @@ export class Terminal { spark.write({ cmd: "size", rows, cols }); } } - // broadcast message to all other clients telling them to close. - this.channel.forEach((spark0, id, _) => { - if (id !== spark.id) { - spark0.write({ cmd: "close" }); - } - }); + // broadcast message to all clients telling them to close, but + // telling requestor to ignore. + spark.write({ cmd: "close", ignore: spark.id }); }; private writeToPty = async (data) => { diff --git a/src/packages/terminal/lib/types.ts b/src/packages/terminal/lib/types.ts index 9ac0c49572..bafc7e1858 100644 --- a/src/packages/terminal/lib/types.ts +++ b/src/packages/terminal/lib/types.ts @@ -20,7 +20,6 @@ export interface Options { export interface PrimusChannel extends EventEmitter { write: (data: object | string) => void; - forEach: (cb: (spark, id, connections) => void) => void; destroy: () => void; // createSpark is not on the real PrimusChannel, but it's part of our mock version for // unit testing in support.ts From 040f373ece1cf9878d2f091c089844be4f99417d Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 29 Jan 2025 17:00:41 +0000 Subject: [PATCH 061/281] nats: work in progress implementing primus style messaging on top of nats --- src/packages/backend/nats/env.ts | 9 + src/packages/backend/nats/index.ts | 1 + src/packages/frontend/client/nats.ts | 43 ++-- .../frontend/project/websocket/api.ts | 22 +- src/packages/nats/primus.ts | 204 ++++++++++++++++++ src/packages/nats/socket.ts | 47 ---- src/packages/project/browser-websocket/api.ts | 13 +- .../project/nats/browser-websocket-api.ts | 4 +- 8 files changed, 269 insertions(+), 74 deletions(-) create mode 100644 src/packages/backend/nats/env.ts create mode 100644 src/packages/nats/primus.ts delete mode 100644 src/packages/nats/socket.ts diff --git a/src/packages/backend/nats/env.ts b/src/packages/backend/nats/env.ts new file mode 100644 index 0000000000..598aac94b3 --- /dev/null +++ b/src/packages/backend/nats/env.ts @@ -0,0 +1,9 @@ +import { sha1 } from "@cocalc/backend/sha1"; +import { JSONCodec } from "nats"; +import { getConnection } from "./index"; + +export async function getEnv() { + const jc = JSONCodec(); + const nc = await getConnection(); + return { nc, jc, sha1 }; +} diff --git a/src/packages/backend/nats/index.ts b/src/packages/backend/nats/index.ts index c7558f0d87..d8a8a669fa 100644 --- a/src/packages/backend/nats/index.ts +++ b/src/packages/backend/nats/index.ts @@ -3,6 +3,7 @@ import { nats } from "@cocalc/backend/data"; import { readFile } from "node:fs/promises"; import getLogger from "@cocalc/backend/logger"; import { connect, credsAuthenticator } from "nats"; +export { getEnv } from "./env"; const logger = getLogger("backend:nats"); diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index c701cfa20b..31b2969387 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -11,7 +11,7 @@ import { parse_query } from "@cocalc/sync/table/util"; import { sha1 } from "@cocalc/util/misc"; import { keys } from "lodash"; import { type HubApi, initHubApi } from "@cocalc/nats/api/index"; -import { Socket } from "@cocalc/nats/socket"; +import { Primus } from "@cocalc/nats/primus"; export class NatsClient { /*private*/ client: WebappClient; @@ -30,7 +30,13 @@ export class NatsClient { getConnection = reuseInFlight(async () => { if (this.nc != null) { - return this.nc; + // undocumented API + if ((this.nc as any).protocol?.isClosed?.()) { + // cause a reconnect. + delete this.nc; + } else { + return this.nc; + } } const server = `${location.protocol == "https:" ? "wss" : "ws"}://${location.host}${appBasePath}/nats`; console.log(`NATS: connecting to ${server}...`); @@ -133,6 +139,14 @@ export class NatsClient { return await js.consumers.get(stream); }; + getEnv = async () => { + return { + sha1, + jc: this.jc, + nc: await this.getConnection(), + }; + }; + synctable = async ( query, options?: { obj?: object; atomic?: boolean; stream?: boolean }, @@ -148,11 +162,7 @@ export class NatsClient { const s = createSyncTable({ ...options, query, - env: { - sha1, - jc: this.jc, - nc: await this.getConnection(), - }, + env: await this.getEnv(), account_id: this.client.account_id, }); await s.init(); @@ -183,11 +193,20 @@ export class NatsClient { return await this.synctable(query, { atomic: true }); }; - createSocket = async (subjects: { listen: string; send: string }) => { - return new Socket({ - ...subjects, - nc: await this.getConnection(), - jc: this.jc, +// createSocket = async (subjects: { listen: string; send: string }) => { +// return new Socket({ +// ...subjects, +// nc: await this.getConnection(), +// jc: this.jc, +// }); +// }; + + primus = async (project_id: string) => { + return new Primus({ + subject: `project.${project_id}.primus`, + env: await this.getEnv(), + role: "client", + id: this.sessionId, }); }; } diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index ccc1eb3a11..85a85eaac0 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -279,17 +279,17 @@ export class API { return this.conn.channel(channel_name); }; - terminal1 = async (path: string, options: object = {}): Promise => { - const subjects = await this.call( - { - cmd: "terminal", - path, - options, - }, - 20000, - ); - return (await webapp_client.nats_client.createSocket(subjects)) as any; - }; +// terminal1 = async (path: string, options: object = {}): Promise => { +// const subjects = await this.call( +// { +// cmd: "terminal", +// path, +// options, +// }, +// 20000, +// ); +// //return (await webapp_client.nats_client.createSocket(subjects)) as any; +// }; project_info = async (): Promise => { const channel_name = await this.primusCall({ cmd: "project_info" }, 60000); diff --git a/src/packages/nats/primus.ts b/src/packages/nats/primus.ts new file mode 100644 index 0000000000..3a73cc086d --- /dev/null +++ b/src/packages/nats/primus.ts @@ -0,0 +1,204 @@ +/* +Implement a websocket as exposed in Primus over NATS. + + +Development: + +1. Change to a directly like packages/project that imports nats and backend + +2. Example session: + +~/cocalc/src/packages/project$ node +... + +Primus = require('@cocalc/nats/primus').Primus; +env = await require('@cocalc/backend/nats').getEnv(); +server = new Primus({subject:'test',env,role:'server',id:'s'}); +sparks = []; server.on("connection", (spark) => sparks.push(spark)) +client = new Primus({subject:'test',env,role:'client',id:'c0'}); + +sparks[0] +client.on('data',(data)=>console.log('client got', data));0 +sparks[0].write("foo") + + +> server.write('foo') +> client2 = new a.Primus({subject:'test',env,role:'client',id:'xx'}); client2.on('data',(data)=>console.log('client2 got', data)); 0 +> server.write('bar') + +> server.on("connection", (spark) => spark.write("hello")) +> client3 = new a.Primus({subject:'test', env, role:'client'}); client3.on('data',(data)=>console.log('client3 got', data)); 0 + + +*/ + +import { EventEmitter } from "events"; +import { type NatsEnv } from "@cocalc/nats/sync/synctable-kv"; + +export type Role = "client" | "server"; + +// function otherRole(role: Role): Role { +// return role == "client" ? "server" : "client"; +// } + +export class Primus extends EventEmitter { + subject: string; + env: NatsEnv; + role: Role; + id: string; + subscribe: string; + subjects: { control: string; server: string; client: string }; + + constructor({ + subject, + env, + role, + id, + }: { + subject: string; + env: NatsEnv; + role: Role; + id: string; + }) { + super(); + this.subject = subject; + this.env = env; + this.role = role; + this.id = id; + this.subjects = { + control: `${subject}.control`, + // only used by client: must agree with spark below + server: `${subject}.server.${id}`, + client: `${subject}.client.${id}`, + }; + if (role == "server") { + this.serve(); + } else { + this.connect(); + } + } + + // channel = (name: string) => { + // return new PrimusChannel({ primus: this, name }); + // }; + + destroy = () => { + // todo + }; + + private serve = async () => { + if (this.role != "server") { + throw Error("only server can serve"); + } + const sub = this.env.nc.subscribe(this.subjects.control); + for await (const mesg of sub) { + const data = this.env.jc.decode(mesg.data) ?? ({} as any); + if (data.cmd == "connect") { + const spark = new Spark({ primus: this, id: data.id }); + this.emit("connection", spark); + mesg.respond(this.env.jc.encode({ status: "ok" })); + } + } + }; + + private connect = async () => { + if (this.role != "client") { + throw Error("only client can connect"); + } + const mesg = this.env.jc.encode({ + cmd: "connect", + id: this.id, + }); + console.log("connecting..."); + await this.env.nc.publish(this.subjects.control, mesg); + console.log("connected:"); + const sub = this.env.nc.subscribe(this.subjects.client); + for await (const mesg of sub) { + const data = this.env.jc.decode(mesg.data) ?? ({} as any); + this.emit("data", data); + } + }; + + write = (data) => { + if (this.role != "client") { + throw Error("only client can write"); + } + this.env.nc.publish(this.subjects.server, this.env.jc.encode({ data })); + }; +} + +// only used on the server +export class Spark extends EventEmitter { + primus: Primus; + id: string; + subjects: { server: string; client: string }; + + constructor({ primus, id }) { + super(); + this.primus = primus; + this.id = id; + const subject = primus.subject; + this.subjects = { + server: `${subject}.server.${id}`, + client: `${subject}.client.${id}`, + }; + this.init(); + } + + private init = async () => { + const sub = this.primus.env.nc.subscribe(this.subjects.server); + for await (const mesg of sub) { + const { data } = this.primus.env.jc.decode(mesg.data) ?? ({} as any); + this.emit("data", data); + } + }; + + write = (data) => { + this.primus.env.nc.publish( + this.subjects.client, + this.primus.env.jc.encode({ data }), + ); + }; + + destroy = () => { + // todo -- maybe call a method on sub created in subscribe? + }; +} + +// export class PrimusChannel extends EventEmitter { +// primus: Primus; +// name: string; +// subjects: { subscribe: string; publish: string }; + +// constructor({ primus, name }) { +// super(); +// this.primus = primus; +// this.name = name; +// const segment = primus.env.sha1(name); +// const base = `${this.primus.subject}.${segment}`; +// this.subjects = { +// subscribe: `${base}.${role}`, +// publish: `${base}.${otherRole(role)}`, +// }; +// this.init(); +// } + +// private init = async () => { +// const sub = this.primus.env.nc.subscribe(this.subjects.subscribe); +// for await (const mesg of sub) { +// const { data } = this.primus.env.jc.decode(mesg.data) ?? ({} as any); +// this.emit("data", data); +// } +// }; + +// write = (data) => { +// this.primus.env.nc.publish( +// this.subjects.publish, +// this.primus.env.jc.encode({ data }), +// ); +// }; + +// destroy = () => { +// // todo +// }; +// } diff --git a/src/packages/nats/socket.ts b/src/packages/nats/socket.ts deleted file mode 100644 index 648e2d01b5..0000000000 --- a/src/packages/nats/socket.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* -Implement a websocket as exposed in Primus over NATS. -*/ - -import { EventEmitter } from "events"; - -export class Socket extends EventEmitter { - private listen: string; - private send: string; - private nc; - private jc; - - constructor({ - listen, - send, - nc, - jc, - }: { - // subject to listen on - listen: string; - // subject to write to - send: string; - // nats connection - nc; - // json codec - jc; - }) { - super(); - this.listen = listen; - this.send = send; - this.nc = nc; - this.jc = jc; - this.startListening(); - } - - private startListening = async () => { - const sub = this.nc.subscribe(this.listen); - for await (const mesg of sub) { - const { data } = this.jc.decode(mesg.data) ?? ({} as any); - this.emit("data", data); - } - }; - - write(data) { - this.nc.publish(this.send, this.jc.encode({ data })); - } -} diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts index 2b8fcfc433..ee26789cfd 100644 --- a/src/packages/project/browser-websocket/api.ts +++ b/src/packages/project/browser-websocket/api.ts @@ -54,7 +54,6 @@ const log = getLogger("websocket-api"); let primus: any = undefined; export function init_websocket_api(_primus: any): void { - primus = _primus; primus.on("connection", function (spark) { @@ -66,7 +65,7 @@ export function init_websocket_api(_primus: any): void { log.debug("primus-api", "request", data, "REQUEST"); const t0 = Date.now(); try { - const resp = await handleApiCall(data, spark); + const resp = await handleApiCall({ data, spark, primus }); //log.debug("primus-api", "response", resp); done(resp); } catch (err) { @@ -93,7 +92,15 @@ export function init_websocket_api(_primus: any): void { }); } -export async function handleApiCall(data: Mesg, spark): Promise { +export async function handleApiCall({ + data, + spark, + primus, +}: { + data: Mesg; + spark; + primus; +}): Promise { const client = getClient(); switch (data.cmd) { case "version": diff --git a/src/packages/project/nats/browser-websocket-api.ts b/src/packages/project/nats/browser-websocket-api.ts index bf379ea1c5..e2e47604f2 100644 --- a/src/packages/project/nats/browser-websocket-api.ts +++ b/src/packages/project/nats/browser-websocket-api.ts @@ -59,8 +59,10 @@ export async function init() { async function handleRequest(data, mesg) { let resp; logger.debug("received cmd:", data?.cmd); + const spark = {} as any; + const primus = {} as any; try { - resp = await handleApiCall(data, {}); + resp = await handleApiCall({ data, spark, primus }); } catch (err) { resp = { error: `${err}` }; } From b8523420a1d9a2ccddf194c2dac057a2d7af9cec Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 29 Jan 2025 18:05:04 +0000 Subject: [PATCH 062/281] nats primus -- implemented basics of sparks. --- src/packages/nats/primus.ts | 170 ++++++++++++++++++++++-------------- 1 file changed, 106 insertions(+), 64 deletions(-) diff --git a/src/packages/nats/primus.ts b/src/packages/nats/primus.ts index 3a73cc086d..423e43561a 100644 --- a/src/packages/nats/primus.ts +++ b/src/packages/nats/primus.ts @@ -21,14 +21,19 @@ sparks[0] client.on('data',(data)=>console.log('client got', data));0 sparks[0].write("foo") +s9 = server.channel('9') +c9 = client.channel('9') +c9.on("data", (data)=>console.log('c9 got', data));0 +s9.on("data", (data)=>console.log('s9 got', data));0 -> server.write('foo') -> client2 = new a.Primus({subject:'test',env,role:'client',id:'xx'}); client2.on('data',(data)=>console.log('client2 got', data)); 0 -> server.write('bar') +c9.write("from client 9") +s9.write("from the server 9") -> server.on("connection", (spark) => spark.write("hello")) -> client3 = new a.Primus({subject:'test', env, role:'client'}); client3.on('data',(data)=>console.log('client3 got', data)); 0 +client_b = new Primus({subject:'test',env,role:'client',id:'cb'}); +c9b = client_b.channel('9') +c9b.on("data", (data)=>console.log('c9b got', data));0 +s9.sparks['cb'].write('blah') */ @@ -41,47 +46,86 @@ export type Role = "client" | "server"; // return role == "client" ? "server" : "client"; // } +function getSubjects({ subject, id, channel, sha1 }) { + // NOTE: when channel is set its sha1 is added as a last + // segment after all of these. + const subjects = { + // control = request/response control channel; clients tell + // server they are connecting via this + control: `${subject}.control`, + // server = a server spark listens on server and client + // publishes to server + server: `${subject}.server.${id}`, + // client = client connection listens on this and + // server spark writes to it + client: `${subject}.client.${id}`, + // channel = when set all clients listen on + // this; server sends to this. + clientChannel: `${subject}.channel.client`, + serverChannel: `${subject}.channel.server`, + }; + if (channel) { + // use sha1 so channel can be any string. + const segment = sha1(channel); + for (const k in subjects) { + subjects[k] += `.${segment}`; + } + } + return subjects; +} + export class Primus extends EventEmitter { subject: string; + channelName: string; env: NatsEnv; role: Role; id: string; subscribe: string; - subjects: { control: string; server: string; client: string }; + sparks: { [id: string]: Spark } = {}; + subjects: { + control: string; + server: string; + client: string; + clientChannel: string; + serverChannel: string; + }; constructor({ subject, + channelName = "", env, role, id, }: { subject: string; + channelName?: string; env: NatsEnv; role: Role; id: string; }) { super(); + this.subject = subject; + this.channelName = channelName; this.env = env; this.role = role; this.id = id; - this.subjects = { - control: `${subject}.control`, - // only used by client: must agree with spark below - server: `${subject}.server.${id}`, - client: `${subject}.client.${id}`, - }; + this.subjects = getSubjects({ + subject, + id, + channel: channelName, + sha1: env.sha1, + }); if (role == "server") { this.serve(); } else { this.connect(); } + if (this.channelName) { + this.subscribeChannel(); + } } - // channel = (name: string) => { - // return new PrimusChannel({ primus: this, name }); - // }; - destroy = () => { // todo }; @@ -94,7 +138,11 @@ export class Primus extends EventEmitter { for await (const mesg of sub) { const data = this.env.jc.decode(mesg.data) ?? ({} as any); if (data.cmd == "connect") { - const spark = new Spark({ primus: this, id: data.id }); + const spark = new Spark({ + primus: this, + id: data.id, + }); + this.sparks[data.id] = spark; this.emit("connection", spark); mesg.respond(this.env.jc.encode({ status: "ok" })); } @@ -119,11 +167,41 @@ export class Primus extends EventEmitter { } }; + private subscribeChannel = async () => { + const subject = + this.role == "client" + ? this.subjects.clientChannel + : this.subjects.serverChannel; + const sub = this.env.nc.subscribe(subject); + for await (const mesg of sub) { + const data = this.env.jc.decode(mesg.data) ?? ({} as any); + this.emit("data", data); + } + }; + + // client: writes to server + // server: write to ALL connected clients in channel model. write = (data) => { - if (this.role != "client") { - throw Error("only client can write"); + let subject; + if (this.role == "server") { + if (!this.channel) { + throw Error("broadcast write not implemented when not in channel mode"); + } + subject = this.subjects.clientChannel; + } else { + subject = this.subjects.serverChannel; } - this.env.nc.publish(this.subjects.server, this.env.jc.encode({ data })); + this.env.nc.publish(subject, this.env.jc.encode({ data })); + }; + + channel = (channelName: string) => { + return new Primus({ + subject: this.subject, + channelName, + env: this.env, + role: this.role, + id: this.id, + }); }; } @@ -131,17 +209,19 @@ export class Primus extends EventEmitter { export class Spark extends EventEmitter { primus: Primus; id: string; - subjects: { server: string; client: string }; + subjects; constructor({ primus, id }) { super(); this.primus = primus; + const { subject, channelName } = primus; this.id = id; - const subject = primus.subject; - this.subjects = { - server: `${subject}.server.${id}`, - client: `${subject}.client.${id}`, - }; + this.subjects = getSubjects({ + subject, + id, + channel: channelName, + sha1: primus.env.sha1, + }); this.init(); } @@ -164,41 +244,3 @@ export class Spark extends EventEmitter { // todo -- maybe call a method on sub created in subscribe? }; } - -// export class PrimusChannel extends EventEmitter { -// primus: Primus; -// name: string; -// subjects: { subscribe: string; publish: string }; - -// constructor({ primus, name }) { -// super(); -// this.primus = primus; -// this.name = name; -// const segment = primus.env.sha1(name); -// const base = `${this.primus.subject}.${segment}`; -// this.subjects = { -// subscribe: `${base}.${role}`, -// publish: `${base}.${otherRole(role)}`, -// }; -// this.init(); -// } - -// private init = async () => { -// const sub = this.primus.env.nc.subscribe(this.subjects.subscribe); -// for await (const mesg of sub) { -// const { data } = this.primus.env.jc.decode(mesg.data) ?? ({} as any); -// this.emit("data", data); -// } -// }; - -// write = (data) => { -// this.primus.env.nc.publish( -// this.subjects.publish, -// this.primus.env.jc.encode({ data }), -// ); -// }; - -// destroy = () => { -// // todo -// }; -// } From acb4a2e633e40c97aaff1d87b9ec2bfbfdef3c04 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 29 Jan 2025 23:10:50 +0000 Subject: [PATCH 063/281] nats: get terminal fully working with "primus over nats" --- src/packages/frontend/client/nats.ts | 18 +++--- .../frontend/project/websocket/api.ts | 28 ++++----- src/packages/nats/primus.ts | 58 +++++++++++++------ .../project/nats/browser-websocket-api.ts | 26 +++++++-- src/packages/terminal/lib/terminal.ts | 4 +- 5 files changed, 83 insertions(+), 51 deletions(-) diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 31b2969387..6dd12a58b2 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -11,7 +11,7 @@ import { parse_query } from "@cocalc/sync/table/util"; import { sha1 } from "@cocalc/util/misc"; import { keys } from "lodash"; import { type HubApi, initHubApi } from "@cocalc/nats/api/index"; -import { Primus } from "@cocalc/nats/primus"; +import { getPrimusConnection } from "@cocalc/nats/primus"; export class NatsClient { /*private*/ client: WebappClient; @@ -193,16 +193,16 @@ export class NatsClient { return await this.synctable(query, { atomic: true }); }; -// createSocket = async (subjects: { listen: string; send: string }) => { -// return new Socket({ -// ...subjects, -// nc: await this.getConnection(), -// jc: this.jc, -// }); -// }; + // createSocket = async (subjects: { listen: string; send: string }) => { + // return new Socket({ + // ...subjects, + // nc: await this.getConnection(), + // jc: this.jc, + // }); + // }; primus = async (project_id: string) => { - return new Primus({ + return getPrimusConnection({ subject: `project.${project_id}.primus`, env: await this.getEnv(), role: "client", diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 85a85eaac0..1b777c3fe7 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -36,7 +36,7 @@ export class API { private project_id: string; private cachedVersion?: number; - constructor(conn: string, project_id: string) { + constructor(conn, project_id: string) { this.conn = conn; this.project_id = project_id; this.listing = reuseInFlight(this.listing.bind(this)); @@ -57,6 +57,11 @@ export class API { }); }; + private getChannel = async (channel_name: string) => { + const natsConn = await webapp_client.nats_client.primus(this.project_id); + return natsConn.channel(channel_name); + }; + call = async (mesg: Mesg, timeout: number) => { try { return await this._call(mesg, timeout); @@ -267,30 +272,17 @@ export class API { }; terminal = async (path: string, options: object = {}): Promise => { - const channel_name = await this.primusCall( + const channel_name = await this.call( { cmd: "terminal", - path: path, + path, options, }, - 60000, + 20000, ); - //console.log(path, "got terminal channel", channel_name); - return this.conn.channel(channel_name); + return await this.getChannel(channel_name) as unknown as Channel; }; -// terminal1 = async (path: string, options: object = {}): Promise => { -// const subjects = await this.call( -// { -// cmd: "terminal", -// path, -// options, -// }, -// 20000, -// ); -// //return (await webapp_client.nats_client.createSocket(subjects)) as any; -// }; - project_info = async (): Promise => { const channel_name = await this.primusCall({ cmd: "project_info" }, 60000); return this.conn.channel(channel_name); diff --git a/src/packages/nats/primus.ts b/src/packages/nats/primus.ts index 423e43561a..d37b0353d0 100644 --- a/src/packages/nats/primus.ts +++ b/src/packages/nats/primus.ts @@ -45,6 +45,26 @@ export type Role = "client" | "server"; // function otherRole(role: Role): Role { // return role == "client" ? "server" : "client"; // } +interface PrimusOptions { + subject: string; + channelName?: string; + env: NatsEnv; + role: Role; + id: string; +} + +const connections: { [key: string]: Primus } = {}; +export function getPrimusConnection(opts: PrimusOptions): Primus { + const key = getKey(opts); + if (connections[key] == null) { + connections[key] = new Primus(opts); + } + return connections[key]; +} + +function getKey({ subject, channelName, role, id }: PrimusOptions) { + return JSON.stringify({ subject, channelName, role, id }); +} function getSubjects({ subject, id, channel, sha1 }) { // NOTE: when channel is set its sha1 is added as a last @@ -54,7 +74,7 @@ function getSubjects({ subject, id, channel, sha1 }) { // server they are connecting via this control: `${subject}.control`, // server = a server spark listens on server and client - // publishes to server + // publishes to server with their id server: `${subject}.server.${id}`, // client = client connection listens on this and // server spark writes to it @@ -89,20 +109,11 @@ export class Primus extends EventEmitter { clientChannel: string; serverChannel: string; }; + // this is just for compat with primus api: + address = { ip: "" }; + conn: { id: string }; - constructor({ - subject, - channelName = "", - env, - role, - id, - }: { - subject: string; - channelName?: string; - env: NatsEnv; - role: Role; - id: string; - }) { + constructor({ subject, channelName = "", env, role, id }: PrimusOptions) { super(); this.subject = subject; @@ -110,6 +121,7 @@ export class Primus extends EventEmitter { this.env = env; this.role = role; this.id = id; + this.conn = { id }; this.subjects = getSubjects({ subject, id, @@ -126,6 +138,12 @@ export class Primus extends EventEmitter { } } + forEach = (f: (spark, id) => void) => { + for (const id in this.sparks) { + f(this.sparks[id], id); + } + }; + destroy = () => { // todo }; @@ -189,9 +207,9 @@ export class Primus extends EventEmitter { } subject = this.subjects.clientChannel; } else { - subject = this.subjects.serverChannel; + subject = this.subjects.server; } - this.env.nc.publish(subject, this.env.jc.encode({ data })); + this.env.nc.publish(subject, this.env.jc.encode(data)); }; channel = (channelName: string) => { @@ -210,12 +228,16 @@ export class Spark extends EventEmitter { primus: Primus; id: string; subjects; + // this is just for compat with primus api: + address = { ip: "" }; + conn: { id: string }; constructor({ primus, id }) { super(); this.primus = primus; const { subject, channelName } = primus; this.id = id; + this.conn = { id }; this.subjects = getSubjects({ subject, id, @@ -228,7 +250,7 @@ export class Spark extends EventEmitter { private init = async () => { const sub = this.primus.env.nc.subscribe(this.subjects.server); for await (const mesg of sub) { - const { data } = this.primus.env.jc.decode(mesg.data) ?? ({} as any); + const data = this.primus.env.jc.decode(mesg.data); this.emit("data", data); } }; @@ -236,7 +258,7 @@ export class Spark extends EventEmitter { write = (data) => { this.primus.env.nc.publish( this.subjects.client, - this.primus.env.jc.encode({ data }), + this.primus.env.jc.encode(data), ); }; diff --git a/src/packages/project/nats/browser-websocket-api.ts b/src/packages/project/nats/browser-websocket-api.ts index e2e47604f2..978489a828 100644 --- a/src/packages/project/nats/browser-websocket-api.ts +++ b/src/packages/project/nats/browser-websocket-api.ts @@ -18,6 +18,16 @@ How to do development (so in a dev project doing cc-in-cc dev): echo 'require("@cocalc/project/client").init(); require("@cocalc/project/nats/browser-websocket-api").init()' | DEBUG=cocalc:* DEBUG_CONSOLE=yes node +Or just run node then paste in + + require("@cocalc/project/client").init(); require("@cocalc/project/nats/browser-websocket-api").init() + +A nice thing about doing that is if you write this deep in some code: + + global.x = { t: this }; + +then after that code runs you can access x from the node console! + 4. Use the browser to see the project is on nats and works: await cc.client.nats_client.projectWebsocketApi({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094', mesg:{cmd:"listing"}}) @@ -33,6 +43,8 @@ import { JSONCodec } from "nats"; import { project_id } from "@cocalc/project/data"; import getConnection from "./connection"; import { handleApiCall } from "@cocalc/project/browser-websocket/api"; +import { getPrimusConnection } from "@cocalc/nats/primus"; +import { sha1 } from "@cocalc/backend/sha1"; const logger = getLogger("project:nats:browser-websocket-api"); @@ -43,6 +55,12 @@ export async function init() { const subject = `project.${project_id}.browser-api`; logger.debug(`initAPI -- NATS project subject '${subject}'`); const sub = nc.subscribe(subject); + const primus = getPrimusConnection({ + subject: `project.${project_id}.primus`, + env: { nc, sha1, jc }, + role: "server", + id: "project", + }); for await (const mesg of sub) { const data = jc.decode(mesg.data) ?? ({} as any); if (data.cmd == "terminate") { @@ -52,17 +70,15 @@ export async function init() { mesg.respond(jc.encode({ exiting: true })); return; } - handleRequest(data, mesg); + handleRequest({ data, mesg, primus }); } } -async function handleRequest(data, mesg) { +async function handleRequest({ data, mesg, primus }) { let resp; logger.debug("received cmd:", data?.cmd); - const spark = {} as any; - const primus = {} as any; try { - resp = await handleApiCall({ data, spark, primus }); + resp = await handleApiCall({ data, spark: {} as any, primus }); } catch (err) { resp = { error: `${err}` }; } diff --git a/src/packages/terminal/lib/terminal.ts b/src/packages/terminal/lib/terminal.ts index 0195fd6501..1049491f4f 100644 --- a/src/packages/terminal/lib/terminal.ts +++ b/src/packages/terminal/lib/terminal.ts @@ -56,6 +56,8 @@ export class Terminal { this.options = { command: DEFAULT_COMMAND, ...options }; this.path = path; this.channel = primus.channel(getChannelName(path)); + console.log(this.channel); + global.x = { t: this }; this.channel.on("connection", this.handleClientConnection); this.remotePtyChannel = primus.channel(getRemotePtyChannelName(path)); this.remotePtyChannel.on("connection", (conn) => { @@ -534,7 +536,7 @@ export class Terminal { spark, data: string | ClientCommand, ) => { - //logger.debug("terminal: browser --> term", name, JSON.stringify(data)); + //logger.debug("terminal: browser --> term", JSON.stringify(data)); if (typeof data === "string") { this.writeToPty(data); } else if (typeof data === "object") { From fd8077c83481568db145fe105c42148a26dd5399 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 30 Jan 2025 00:43:46 +0000 Subject: [PATCH 064/281] nats primus: add ping interval and implement auto destroy on server side --- .../frontend/project/websocket/api.ts | 9 +- src/packages/nats/package.json | 13 ++- src/packages/nats/primus.ts | 101 ++++++++++++++++-- src/packages/pnpm-lock.yaml | 3 + 4 files changed, 112 insertions(+), 14 deletions(-) diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 1b777c3fe7..9485b19801 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -59,7 +59,8 @@ export class API { private getChannel = async (channel_name: string) => { const natsConn = await webapp_client.nats_client.primus(this.project_id); - return natsConn.channel(channel_name); + // TODO -- typing + return natsConn.channel(channel_name) as Channel; }; call = async (mesg: Mesg, timeout: number) => { @@ -280,12 +281,12 @@ export class API { }, 20000, ); - return await this.getChannel(channel_name) as unknown as Channel; + return await this.getChannel(channel_name); }; project_info = async (): Promise => { - const channel_name = await this.primusCall({ cmd: "project_info" }, 60000); - return this.conn.channel(channel_name); + const channel_name = await this.call({ cmd: "project_info" }, 60000); + return await this.getChannel(channel_name); }; // Get the lean *channel* for the given '.lean' path. diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 5e2877fb60..159efd105d 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -15,15 +15,24 @@ "test": "pnpm exec jest", "prepublishOnly": "pnpm test" }, - "files": ["dist/**", "README.md", "package.json"], + "files": [ + "dist/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["utilities", "nats", "cocalc"], + "keywords": [ + "utilities", + "nats", + "cocalc" + ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/nats": "workspace:*", "@cocalc/util": "workspace:*", "@nats-io/jetstream": "3.0.0-36", "@nats-io/kv": "3.0.0-30", + "awaiting": "^3.0.0", "events": "3.3.0", "json-stable-stringify": "^1.0.1", "lodash": "^4.17.21", diff --git a/src/packages/nats/primus.ts b/src/packages/nats/primus.ts index d37b0353d0..3bee608632 100644 --- a/src/packages/nats/primus.ts +++ b/src/packages/nats/primus.ts @@ -39,9 +39,12 @@ s9.sparks['cb'].write('blah') import { EventEmitter } from "events"; import { type NatsEnv } from "@cocalc/nats/sync/synctable-kv"; +import { delay } from "awaiting"; export type Role = "client" | "server"; +const PING_INTERVAL = 10000; + // function otherRole(role: Role): Role { // return role == "client" ? "server" : "client"; // } @@ -94,6 +97,8 @@ function getSubjects({ subject, id, channel, sha1 }) { return subjects; } +type State = "ready" | "closed"; + export class Primus extends EventEmitter { subject: string; channelName: string; @@ -112,6 +117,11 @@ export class Primus extends EventEmitter { // this is just for compat with primus api: address = { ip: "" }; conn: { id: string }; + subs: any[] = []; + OPEN = 1; + CLOSE = 0; + readyState: 0; + state: State = "ready"; constructor({ subject, channelName = "", env, role, id }: PrimusOptions) { super(); @@ -131,7 +141,7 @@ export class Primus extends EventEmitter { if (role == "server") { this.serve(); } else { - this.connect(); + this.client(); } if (this.channelName) { this.subscribeChannel(); @@ -145,17 +155,42 @@ export class Primus extends EventEmitter { }; destroy = () => { - // todo + if (this.state == "closed") { + return; + } + this.state = "closed"; + delete connections[getKey(this)]; + for (const sub of this.subs) { + sub.close(); + } + this.subs = []; + for (const id in this.sparks) { + this.sparks[id].destroy(); + } + this.sparks = {}; }; + end = () => this.destroy(); + + close = () => this.destroy(); + + connect = () => {}; + private serve = async () => { if (this.role != "server") { throw Error("only server can serve"); } + this.deleteSparks(); const sub = this.env.nc.subscribe(this.subjects.control); + this.subs.push(sub); for await (const mesg of sub) { const data = this.env.jc.decode(mesg.data) ?? ({} as any); - if (data.cmd == "connect") { + if (data.cmd == "ping") { + const spark = this.sparks[data.id]; + if (spark != null) { + spark.lastPing = Date.now(); + } + } else if (data.cmd == "connect") { const spark = new Spark({ primus: this, id: data.id, @@ -167,7 +202,19 @@ export class Primus extends EventEmitter { } }; - private connect = async () => { + private deleteSparks = async () => { + while (this.state != "closed") { + for (const id in this.sparks) { + const spark = this.sparks[id]; + if (Date.now() - spark.lastPing > PING_INTERVAL * 1.5) { + spark.destroy(); + } + } + await delay(PING_INTERVAL * 1.5); + } + }; + + private client = async () => { if (this.role != "client") { throw Error("only client can connect"); } @@ -177,20 +224,42 @@ export class Primus extends EventEmitter { }); console.log("connecting..."); await this.env.nc.publish(this.subjects.control, mesg); + this.clientPing(); console.log("connected:"); const sub = this.env.nc.subscribe(this.subjects.client); + this.subs.push(sub); for await (const mesg of sub) { const data = this.env.jc.decode(mesg.data) ?? ({} as any); this.emit("data", data); } }; + private clientPing = async () => { + while (this.state != "closed") { + try { + await this.env.nc.publish( + this.subjects.control, + this.env.jc.encode({ + cmd: "ping", + id: this.id, + }), + ); + } catch { + // if ping fails, connection is not working, so die. + this.destroy(); + return; + } + await delay(PING_INTERVAL); + } + }; + private subscribeChannel = async () => { const subject = this.role == "client" ? this.subjects.clientChannel : this.subjects.serverChannel; const sub = this.env.nc.subscribe(subject); + this.subs.push(sub); for await (const mesg of sub) { const data = this.env.jc.decode(mesg.data) ?? ({} as any); this.emit("data", data); @@ -210,6 +279,7 @@ export class Primus extends EventEmitter { subject = this.subjects.server; } this.env.nc.publish(subject, this.env.jc.encode(data)); + return true; }; channel = (channelName: string) => { @@ -228,9 +298,12 @@ export class Spark extends EventEmitter { primus: Primus; id: string; subjects; + lastPing = Date.now(); // this is just for compat with primus api: address = { ip: "" }; conn: { id: string }; + subs: any[] = []; + state: State = "ready"; constructor({ primus, id }) { super(); @@ -247,8 +320,23 @@ export class Spark extends EventEmitter { this.init(); } + destroy = () => { + if (this.state == "closed") { + return; + } + this.state = "closed"; + for (const sub of this.subs) { + sub.close(); + } + this.subs = []; + delete this.primus.sparks[this.id]; + }; + + end = () => this.destroy(); + private init = async () => { const sub = this.primus.env.nc.subscribe(this.subjects.server); + this.subs.push(sub); for await (const mesg of sub) { const data = this.primus.env.jc.decode(mesg.data); this.emit("data", data); @@ -260,9 +348,6 @@ export class Spark extends EventEmitter { this.subjects.client, this.primus.env.jc.encode(data), ); - }; - - destroy = () => { - // todo -- maybe call a method on sub created in subscribe? + return true; }; } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index b770b328f2..57dacd66e2 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1063,6 +1063,9 @@ importers: '@nats-io/kv': specifier: 3.0.0-30 version: 3.0.0-30 + awaiting: + specifier: ^3.0.0 + version: 3.0.0 events: specifier: 3.3.0 version: 3.3.0 From 445d0483cb1eeb4fea2bdd318316585807c4d118 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 30 Jan 2025 01:55:36 +0000 Subject: [PATCH 065/281] nats: primus -- going to give upon this approach --- src/packages/frontend/project/websocket/api.ts | 2 +- src/packages/nats/primus.ts | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 9485b19801..0eca13304b 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -60,7 +60,7 @@ export class API { private getChannel = async (channel_name: string) => { const natsConn = await webapp_client.nats_client.primus(this.project_id); // TODO -- typing - return natsConn.channel(channel_name) as Channel; + return natsConn.channel(channel_name) as unknown as Channel; }; call = async (mesg: Mesg, timeout: number) => { diff --git a/src/packages/nats/primus.ts b/src/packages/nats/primus.ts index 3bee608632..048521010f 100644 --- a/src/packages/nats/primus.ts +++ b/src/packages/nats/primus.ts @@ -60,7 +60,10 @@ const connections: { [key: string]: Primus } = {}; export function getPrimusConnection(opts: PrimusOptions): Primus { const key = getKey(opts); if (connections[key] == null) { + console.log("getPrimus", key, "CREATING", opts); connections[key] = new Primus(opts); + } else { + console.log("getPrimus", key, "already have it", opts); } return connections[key]; } @@ -126,6 +129,12 @@ export class Primus extends EventEmitter { constructor({ subject, channelName = "", env, role, id }: PrimusOptions) { super(); + console.log("PRIMUS Creating", { + subject, + id, + channel: channelName, + }); + this.subject = subject; this.channelName = channelName; this.env = env; @@ -159,6 +168,7 @@ export class Primus extends EventEmitter { return; } this.state = "closed"; + console.log("destroy", getKey(this)); delete connections[getKey(this)]; for (const sub of this.subs) { sub.close(); @@ -283,7 +293,7 @@ export class Primus extends EventEmitter { }; channel = (channelName: string) => { - return new Primus({ + return getPrimusConnection({ subject: this.subject, channelName, env: this.env, From e053a9d5702d4a18ac032abcfdcbdaa05b67e4d2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 30 Jan 2025 14:27:52 +0000 Subject: [PATCH 066/281] nats: enable by default new project api and terminal using nats --- .../terminal-editor/connected-terminal.ts | 4 +- .../project/browser-websocket/server.ts | 4 +- src/packages/project/nats/api.ts | 112 ++++++++++++++++++ src/packages/project/nats/index.ts | 101 ++-------------- src/packages/server/nats/auth.ts | 9 +- src/packages/terminal/lib/terminal.ts | 2 - 6 files changed, 134 insertions(+), 98 deletions(-) create mode 100644 src/packages/project/nats/api.ts diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index ba96ded7c3..c82ed2eac8 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -11,7 +11,7 @@ extra support for being connected to: - frame-editor (via actions) */ -const USE_NATS = false; +const USE_NATS = true; import { callback, delay } from "awaiting"; import { Map } from "immutable"; @@ -301,7 +301,7 @@ export class Terminal { }; async connect(): Promise { - if (USE_NATS && this.path.startsWith("nats/")) { + if (USE_NATS) { return await this.connectNats(); } this.assert_not_closed(); diff --git a/src/packages/project/browser-websocket/server.ts b/src/packages/project/browser-websocket/server.ts index 81390ba4c6..93c6f3a79c 100644 --- a/src/packages/project/browser-websocket/server.ts +++ b/src/packages/project/browser-websocket/server.ts @@ -12,7 +12,7 @@ import { Router } from "express"; import { Server } from "http"; import Primus from "primus"; import type { PrimusWithChannels } from "@cocalc/terminal"; -import { init as initNatsBrowserWebsocketApi } from "@cocalc/project/nats/browser-websocket-api"; +import initNats from "@cocalc/project/nats"; // We are NOT using UglifyJS because it can easily take 3 blocking seconds of cpu // during project startup to save 100kb -- it just isn't worth it. Obviously, it @@ -58,7 +58,7 @@ export default function init(server: Server, basePath: string): Router { ); // we also init the new nats server, which is meant to replace this: - initNatsBrowserWebsocketApi(); + initNats(); return router; } diff --git a/src/packages/project/nats/api.ts b/src/packages/project/nats/api.ts new file mode 100644 index 0000000000..535e855ccf --- /dev/null +++ b/src/packages/project/nats/api.ts @@ -0,0 +1,112 @@ +/* +How to do development (so in a dev project doing cc-in-cc dev). + +0. From the browser, terminate this api server running in the project already, if any + + await cc.client.nats_client.project({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e', endpoint:"terminate"}) + +1. Open a terminal in the project itself, which sets up the required environment variables, e.g., + + - COCALC_NATS_JWT -- this has the valid JWT issued to grant the project rights to use nats + - COCALC_PROJECT_ID + +You can type the following into the miniterminal in a project and copy the output into a terminal here to +setup the same environment and make starting this server act like this part of a project. + + export | grep -E "COCALC|HOME" + +2. Do this: + + echo 'require("@cocalc/project/nats/api").init()' | DEBUG=cocalc:* DEBUG_CONSOLE=yes node + +or just run node and paste + + require("@cocalc/project/nats/api").init() + +if you want to easily be able to grab some state, e.g., global.x = {...} in some code. + +5. Use the browser to see the project is on nats and works: + + await cc.client.nats_client.project({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e', endpoint:"exec", params:{command:'echo $COCALC_PROJECT_ID'}}) + +*/ + +import { getLogger } from "@cocalc/project/logger"; +import { JSONCodec } from "nats"; +import { project_id } from "@cocalc/project/data"; +import { handleExecShellCode } from "@cocalc/project/exec_shell_code"; +import { realpath } from "@cocalc/project/browser-websocket/realpath"; +import getConnection from "./connection"; + +const logger = getLogger("project:nats"); + +const jc = JSONCodec(); + +export async function init() { + const nc = await getConnection(); + const subject = `project.${project_id}.api.>`; + logger.debug(`initAPI -- NATS project subject '${subject}'`); + const sub = nc.subscribe(subject); + for await (const mesg of sub) { + const request = jc.decode(mesg.data) ?? {} as any; + if (request.endpoint == "terminate") { + mesg.respond(jc.encode({ status: "terminate" })); + // @ts-ignore + sub.close(); + return; + } + handleRequest(request, mesg, nc); + } +} + +async function handleRequest(request, mesg, nc) { + const segments = mesg.subject.split("."); + const group = segments[3]; // 'owner', 'collaborator', etc. + const account_id = segments[4]; + return await handleApiRequest({ request, mesg, group, account_id, nc }); +} + +async function handleApiRequest({ request, mesg, group, account_id, nc }) { + let resp; + try { + const { endpoint, params } = request; + logger.debug("handling project request:", { + endpoint, + params, + group, + account_id, + }); + resp = await getResponse({ endpoint, params, nc }); + } catch (err) { + resp = { error: `${err}` }; + } + logger.debug("responding with ", resp); + mesg.respond(jc.encode(resp)); +} + +import { + createTerminal, + restartTerminal, + terminalCommand, + writeToTerminal, +} from "./terminal"; +async function getResponse({ endpoint, params, nc }) { + switch (endpoint) { + case "ping": + return { pong: Date.now() }; + case "realpath": + return realpath(params.path); + case "exec": + return await handleExecShellCode(params); + case "create-terminal": + return await createTerminal({ params, nc }); + case "restart-terminal": + return await restartTerminal(params); + case "terminal-command": + return await terminalCommand(params); + case "write-to-terminal": + return await writeToTerminal(params); + default: + throw Error(`unknown endpoint '${endpoint}'`); + } +} diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts index e740476d58..2f98d24607 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/nats/index.ts @@ -1,96 +1,17 @@ /* -How to do development (so in a dev project doing cc-in-cc dev). - -1. Open a terminal in the project itself, which sets up the required environment variables, e.g., - - COCALC_NATS_JWT -- this has the valid JWT issued to grant the project rights to use nats - - COCALC_PROJECT_ID - -2. cd to your dev packages/project source code, e.g., ../cocalc/src/packages/project - -3. Do this: - - echo 'require("@cocalc/project/nats").default()' | DEBUG=cocalc:* DEBUG_CONSOLE=yes node - -4. Use the browser to see the project is on nats and works: - - await cc.client.nats_client.project({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e', endpoint:"exec", params:{command:'echo $COCALC_PROJECT_ID'}}) +Start the NATS servers: +- the new api +- legacy api */ import { getLogger } from "@cocalc/project/logger"; -import { JSONCodec } from "nats"; -import { project_id } from "@cocalc/project/data"; -import { handleExecShellCode } from "@cocalc/project/exec_shell_code"; -import { realpath } from "@cocalc/project/browser-websocket/realpath"; -import getConnection from "./connection"; - -const logger = getLogger("project:nats"); - -export default async function initNatsServer() { - const nc = await getConnection(); - initAPI(nc); -} - -const jc = JSONCodec(); - -export async function initAPI(nc) { - const subject = `project.${project_id}.api.>`; - logger.debug(`initAPI -- NATS project subject '${subject}'`); - const sub = nc.subscribe(subject); - for await (const mesg of sub) { - handleRequest(mesg, nc); - } -} - -async function handleRequest(mesg, nc) { - const segments = mesg.subject.split("."); - const group = segments[3]; // 'owner', 'collaborator', etc. - const account_id = segments[4]; - await handleApiRequest({ mesg, group, account_id, nc }); -} - -async function handleApiRequest({ mesg, group, account_id, nc }) { - const request = jc.decode(mesg.data) ?? {}; - let resp; - try { - const { endpoint, params } = request as any; - logger.debug("handling project request:", { - endpoint, - params, - group, - account_id, - }); - resp = await getResponse({ endpoint, params, nc }); - } catch (err) { - resp = { error: `${err}` }; - } - logger.debug("responding with ", resp); - mesg.respond(jc.encode(resp)); -} - -import { - createTerminal, - restartTerminal, - terminalCommand, - writeToTerminal, -} from "./terminal"; -async function getResponse({ endpoint, params, nc }) { - switch (endpoint) { - case "ping": - return { pong: Date.now() }; - case "realpath": - return realpath(params.path); - case "exec": - return await handleExecShellCode(params); - case "create-terminal": - return await createTerminal({ params, nc }); - case "restart-terminal": - return await restartTerminal(params); - case "terminal-command": - return await terminalCommand(params); - case "write-to-terminal": - return await writeToTerminal(params); - default: - throw Error(`unknown endpoint '${endpoint}'`); - } +const logger = getLogger("project:nats:index"); +import { init as initAPI } from "./api"; +import { init as initWebsocketApi } from "./browser-websocket-api"; + +export default async function init() { + logger.debug("starting NATS project servers"); + initAPI(); + initWebsocketApi(); } diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index d67a47a404..1c91ce9fdd 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -108,8 +108,13 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { throw Error("must be a valid uuid"); } const userType = getCoCalcUserType(cocalcUser); - const goalPub = new Set(["_INBOX.>", `hub.${userType}.${userId}.>`]); - const goalSub = new Set(["_INBOX.>"]); + // TODO: jetstream permissions are WAY TO BROAD. + const goalPub = new Set([ + "_INBOX.>", + `hub.${userType}.${userId}.>`, + "$JS.API.>", + ]); + const goalSub = new Set(["_INBOX.>", "$JS.API.>"]); if (userType == "account") { goalSub.add(`$KV.account-${userId}.>`); diff --git a/src/packages/terminal/lib/terminal.ts b/src/packages/terminal/lib/terminal.ts index 1049491f4f..3cd6c73759 100644 --- a/src/packages/terminal/lib/terminal.ts +++ b/src/packages/terminal/lib/terminal.ts @@ -56,8 +56,6 @@ export class Terminal { this.options = { command: DEFAULT_COMMAND, ...options }; this.path = path; this.channel = primus.channel(getChannelName(path)); - console.log(this.channel); - global.x = { t: this }; this.channel.on("connection", this.handleClientConnection); this.remotePtyChannel = primus.channel(getRemotePtyChannelName(path)); this.remotePtyChannel.on("connection", (conn) => { From c74abd93d42be278a5962953fdb67ba25f542b29 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 30 Jan 2025 16:15:07 +0000 Subject: [PATCH 067/281] nats: creating typescript project RPC framework --- src/packages/frontend/client/nats.ts | 53 ++++++++++- src/packages/nats/{api => hub-api}/db.ts | 0 src/packages/nats/{api => hub-api}/index.ts | 0 .../nats/{api => hub-api}/purchases.ts | 0 src/packages/nats/{api => hub-api}/system.ts | 0 src/packages/nats/{api => hub-api}/util.ts | 0 src/packages/nats/package.json | 17 ++-- src/packages/nats/project-api/index.ts | 26 ++++++ src/packages/nats/project-api/system.ts | 7 ++ src/packages/project/nats/api/index.ts | 89 +++++++++++++++++++ src/packages/project/nats/api/system.ts | 3 + src/packages/server/nats/api/index.ts | 2 +- 12 files changed, 182 insertions(+), 15 deletions(-) rename src/packages/nats/{api => hub-api}/db.ts (100%) rename src/packages/nats/{api => hub-api}/index.ts (100%) rename src/packages/nats/{api => hub-api}/purchases.ts (100%) rename src/packages/nats/{api => hub-api}/system.ts (100%) rename src/packages/nats/{api => hub-api}/util.ts (100%) create mode 100644 src/packages/nats/project-api/index.ts create mode 100644 src/packages/nats/project-api/system.ts create mode 100644 src/packages/project/nats/api/index.ts create mode 100644 src/packages/project/nats/api/system.ts diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 6dd12a58b2..701588d99f 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -10,7 +10,8 @@ import { randomId } from "@cocalc/nats/util"; import { parse_query } from "@cocalc/sync/table/util"; import { sha1 } from "@cocalc/util/misc"; import { keys } from "lodash"; -import { type HubApi, initHubApi } from "@cocalc/nats/api/index"; +import { type HubApi, initHubApi } from "@cocalc/nats/hub-api"; +import { type ProjectApi, initProjectApi } from "@cocalc/nats/project-api"; import { getPrimusConnection } from "@cocalc/nats/primus"; export class NatsClient { @@ -71,10 +72,12 @@ export class NatsClient { service = "api", name, args = [], + timeout = 5000, }: { service?: string; name: string; args: any[]; + timeout?: number; }) => { const nc = await this.getConnection(); const subject = `hub.account.${this.client.account_id}.${service}`; @@ -84,7 +87,53 @@ export class NatsClient { name, args, }), - { timeout: 5000 }, + { timeout }, + ); + return this.jc.decode(resp.data); + }; + + // Returns api for RPC calls to the project with typescript support! + projectApi = ({ + project_id, + timeout, + }: { + project_id: string; + timeout?: number; + }): ProjectApi => { + const callProjectApi = async ({ name, args }) => { + return await this.callProject({ + project_id, + timeout, + service: "api", + name, + args, + }); + }; + return initProjectApi(callProjectApi); + }; + + private callProject = async ({ + service = "api", + project_id, + name, + args = [], + timeout = 5000, + }: { + service?: string; + project_id: string; + name: string; + args: any[]; + timeout?: number; + }) => { + const nc = await this.getConnection(); + const subject = `project.${project_id}.${service}`; + const resp = await nc.request( + subject, + this.jc.encode({ + name, + args, + }), + { timeout }, ); return this.jc.decode(resp.data); }; diff --git a/src/packages/nats/api/db.ts b/src/packages/nats/hub-api/db.ts similarity index 100% rename from src/packages/nats/api/db.ts rename to src/packages/nats/hub-api/db.ts diff --git a/src/packages/nats/api/index.ts b/src/packages/nats/hub-api/index.ts similarity index 100% rename from src/packages/nats/api/index.ts rename to src/packages/nats/hub-api/index.ts diff --git a/src/packages/nats/api/purchases.ts b/src/packages/nats/hub-api/purchases.ts similarity index 100% rename from src/packages/nats/api/purchases.ts rename to src/packages/nats/hub-api/purchases.ts diff --git a/src/packages/nats/api/system.ts b/src/packages/nats/hub-api/system.ts similarity index 100% rename from src/packages/nats/api/system.ts rename to src/packages/nats/hub-api/system.ts diff --git a/src/packages/nats/api/util.ts b/src/packages/nats/hub-api/util.ts similarity index 100% rename from src/packages/nats/api/util.ts rename to src/packages/nats/hub-api/util.ts diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 159efd105d..13a53ef247 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -4,8 +4,9 @@ "description": "CoCalc NATS integration code. Usable by both nodejs and browser.", "exports": { "./sync/*": "./dist/sync/*.js", - "./api": "./dist/api/index.js", - "./api/*": "./dist/api/*.js", + "./hub-api": "./dist/hub-api/index.js", + "./hub-api/*": "./dist/hub-api/*.js", + "./project-api": "./dist/project-api/index.js", "./*": "./dist/*.js" }, "scripts": { @@ -15,17 +16,9 @@ "test": "pnpm exec jest", "prepublishOnly": "pnpm test" }, - "files": [ - "dist/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "nats", - "cocalc" - ], + "keywords": ["utilities", "nats", "cocalc"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/nats": "workspace:*", diff --git a/src/packages/nats/project-api/index.ts b/src/packages/nats/project-api/index.ts new file mode 100644 index 0000000000..eb2d40104c --- /dev/null +++ b/src/packages/nats/project-api/index.ts @@ -0,0 +1,26 @@ +import { type System, system } from "./system"; + +export interface ProjectApi { + system: System; +} + +const ProjectApiStructure = { + system, +} as const; + +export function initProjectApi(callProjectApi): ProjectApi { + const projectApi: any = {}; + for (const group in ProjectApiStructure) { + if (projectApi[group] == null) { + projectApi[group] = {}; + } + for (const functionName in ProjectApiStructure[group]) { + projectApi[group][functionName] = async (...args) => + await callProjectApi({ + name: `${group}.${functionName}`, + args, + }); + } + } + return projectApi as ProjectApi; +} diff --git a/src/packages/nats/project-api/system.ts b/src/packages/nats/project-api/system.ts new file mode 100644 index 0000000000..0d5a5d585e --- /dev/null +++ b/src/packages/nats/project-api/system.ts @@ -0,0 +1,7 @@ +export const system = { + ping: true, +}; + +export interface System { + ping: () => { now: number }; +} diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts new file mode 100644 index 0000000000..e1d23e247d --- /dev/null +++ b/src/packages/project/nats/api/index.ts @@ -0,0 +1,89 @@ +/* +How to do development (so in a dev project doing cc-in-cc dev). + +0. From the browser, terminate this api server running in the project already, if any + + await cc.client.nats_client.projectApi({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}).terminate() + +1. Open a terminal in the project itself, which sets up the required environment variables, e.g., + + - COCALC_NATS_JWT -- this has the valid JWT issued to grant the project rights to use nats + - COCALC_PROJECT_ID + +You can type the following into the miniterminal in a project and copy the output into a terminal here to +setup the same environment and make starting this server act like this part of a project. + + export | grep -E "COCALC|HOME" + +2. Do this: + + echo 'require("@cocalc/project/nats/api/index").init()' | DEBUG=cocalc:* DEBUG_CONSOLE=yes node + +or just run node and paste + + require("@cocalc/project/nats/api").init() + +if you want to easily be able to grab some state, e.g., global.x = {...} in some code. + +5. Use the browser to see the project is on nats and works: + + a = cc.client.nats_client.projectApi({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}); + await a.system.ping(); + await a.system.exec({command:'echo $COCALC_PROJECT_ID'}); + +*/ + +import { JSONCodec } from "nats"; +import getLogger from "@cocalc/backend/logger"; +import { type ProjectApi } from "@cocalc/nats/project-api"; +import { getConnection } from "@cocalc/backend/nats"; +import { project_id } from "@cocalc/project/data"; + +const logger = getLogger("project:nats:api"); + +const jc = JSONCodec(); + +export async function init() { + const subject = `project.${project_id}.api`; + logger.debug(`initAPI -- subject='${subject}', options=`, { + queue: "0", + }); + const nc = await getConnection(); + const sub = nc.subscribe(subject, { queue: "0" }); + for await (const mesg of sub) { + const request = jc.decode(mesg.data) ?? ({} as any); + if (request.name == "terminate") { + // special hook so admin can terminate handling. This is useful for development. + mesg.respond(jc.encode({ status: "terminating" })); + return; + } + handleApiRequest(request, mesg); + } +} + +async function handleApiRequest(request, mesg) { + let resp; + try { + const { name, args } = request as any; + logger.debug("handling project.api request:", { name }); + resp = (await getResponse({ name, args })) ?? null; + } catch (err) { + resp = { error: `${err}` }; + } + mesg.respond(jc.encode(resp)); +} + +import * as system from "./system"; + +export const projectApi: ProjectApi = { + system, +}; + +async function getResponse({ name, args }) { + const [group, functionName] = name.split("."); + const f = projectApi[group]?.[functionName]; + if (f == null) { + throw Error(`unknown function '${name}'`); + } + return await f(...args); +} diff --git a/src/packages/project/nats/api/system.ts b/src/packages/project/nats/api/system.ts new file mode 100644 index 0000000000..19d6eef179 --- /dev/null +++ b/src/packages/project/nats/api/system.ts @@ -0,0 +1,3 @@ +export function ping() { + return { now: Date.now() }; +} diff --git a/src/packages/server/nats/api/index.ts b/src/packages/server/nats/api/index.ts index 47bc85a31c..82f617a060 100644 --- a/src/packages/server/nats/api/index.ts +++ b/src/packages/server/nats/api/index.ts @@ -36,7 +36,7 @@ To view all requests (and replies) in realtime: import { JSONCodec } from "nats"; import getLogger from "@cocalc/backend/logger"; -import { type HubApi, getUserId, transformArgs } from "@cocalc/nats/api/index"; +import { type HubApi, getUserId, transformArgs } from "@cocalc/nats/hub-api"; import { getConnection } from "@cocalc/backend/nats"; import userIsInGroup from "@cocalc/server/accounts/is-in-group"; From 077c2c40aadb7e7a83a8acd3006ae3c7b32218a8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 30 Jan 2025 18:06:57 +0000 Subject: [PATCH 068/281] nats: start newest project api as part of starting project and add terminal support --- .../nats-terminal-connection.ts | 24 ++-- src/packages/nats/hub-api/index.ts | 6 +- src/packages/nats/project-api/index.ts | 14 ++- src/packages/nats/project-api/system.ts | 13 +- src/packages/nats/project-api/terminal.ts | 13 ++ src/packages/nats/util.ts | 11 ++ .../project/browser-websocket/exec-code.ts | 7 +- src/packages/project/exec_shell_code.ts | 2 +- src/packages/project/nats/api.ts | 112 ------------------ src/packages/project/nats/api/index.ts | 24 ++-- src/packages/project/nats/api/system.ts | 9 +- src/packages/project/nats/api/terminal.ts | 8 ++ src/packages/project/nats/connection.ts | 2 +- src/packages/project/nats/index.ts | 5 +- src/packages/project/nats/terminal.ts | 6 +- src/packages/util/types/execute-code.ts | 4 + 16 files changed, 104 insertions(+), 156 deletions(-) create mode 100644 src/packages/nats/project-api/terminal.ts delete mode 100644 src/packages/project/nats/api.ts create mode 100644 src/packages/project/nats/api/terminal.ts diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index db89ee8e68..aefef536da 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -21,6 +21,7 @@ export class NatsTerminalConnection extends EventEmitter { private terminalResize; private openPaths; private closePaths; + private project; constructor({ project_id, @@ -44,7 +45,8 @@ export class NatsTerminalConnection extends EventEmitter { this.keep = keep; this.openPaths = openPaths; this.closePaths = closePaths; - // move to util so guaranteed in sync with project + this.project = webapp_client.nats_client.projectApi({ project_id }); + // TODO: move to @cocalc/nats (?) so guaranteed in sync with project this.subject = `project.${project_id}.terminal.${sha1(path)}`; this.cmd_subject = `project.${project_id}.terminal-cmd.${sha1(path)}`; } @@ -73,18 +75,14 @@ export class NatsTerminalConnection extends EventEmitter { return; } } - await webapp_client.nats_client.project({ - project_id: this.project_id, - endpoint: "terminal-command", - params: { path: this.path, ...data, client }, - }); + await this.project.terminal.command({ path: this.path, ...data, client }); return; } const f = async () => { - await webapp_client.nats_client.project({ - project_id: this.project_id, - endpoint: "write-to-terminal", - params: { path: this.path, data, keep: this.keep }, + await this.project.terminal.write({ + path: this.path, + data, + keep: this.keep, }); }; @@ -103,11 +101,7 @@ export class NatsTerminalConnection extends EventEmitter { private start = reuseInFlight(async () => { // ensure running: - await webapp_client.nats_client.project({ - project_id: this.project_id, - endpoint: "create-terminal", - params: { path: this.path }, - }); + await this.project.terminal.create({ path: this.path }); }); private getConsumer = async () => { diff --git a/src/packages/nats/hub-api/index.ts b/src/packages/nats/hub-api/index.ts index 1efa9164f8..d2c9e3ff56 100644 --- a/src/packages/nats/hub-api/index.ts +++ b/src/packages/nats/hub-api/index.ts @@ -2,6 +2,7 @@ import { isValidUUID } from "@cocalc/util/misc"; import { type Purchases, purchases } from "./purchases"; import { type System, system } from "./system"; import { type DB, db } from "./db"; +import { handleErrorMessage } from "@cocalc/nats/util"; export interface HubApi { system: System; @@ -9,7 +10,6 @@ export interface HubApi { purchases: Purchases; } - const HubApiStructure = { system, db, @@ -33,7 +33,9 @@ export function initHubApi(callHubApi): HubApi { } for (const functionName in HubApiStructure[group]) { hubApi[group][functionName] = async (...args) => - await callHubApi({ name: `${group}.${functionName}`, args }); + handleErrorMessage( + await callHubApi({ name: `${group}.${functionName}`, args }), + ); } } return hubApi as HubApi; diff --git a/src/packages/nats/project-api/index.ts b/src/packages/nats/project-api/index.ts index eb2d40104c..ffaf939955 100644 --- a/src/packages/nats/project-api/index.ts +++ b/src/packages/nats/project-api/index.ts @@ -1,11 +1,15 @@ import { type System, system } from "./system"; +import { type Terminal, terminal } from "./terminal"; +import { handleErrorMessage} from "@cocalc/nats/util"; export interface ProjectApi { system: System; + terminal: Terminal; } const ProjectApiStructure = { system, + terminal, } as const; export function initProjectApi(callProjectApi): ProjectApi { @@ -16,10 +20,12 @@ export function initProjectApi(callProjectApi): ProjectApi { } for (const functionName in ProjectApiStructure[group]) { projectApi[group][functionName] = async (...args) => - await callProjectApi({ - name: `${group}.${functionName}`, - args, - }); + handleErrorMessage( + await callProjectApi({ + name: `${group}.${functionName}`, + args, + }), + ); } } return projectApi as ProjectApi; diff --git a/src/packages/nats/project-api/system.ts b/src/packages/nats/project-api/system.ts index 0d5a5d585e..b66ca6c279 100644 --- a/src/packages/nats/project-api/system.ts +++ b/src/packages/nats/project-api/system.ts @@ -1,7 +1,18 @@ +import type { + ExecuteCodeOutput, + ExecuteCodeOptions, +} from "@cocalc/util/types/execute-code"; + export const system = { ping: true, + terminate: true, + exec: true, + realpath: true, }; export interface System { - ping: () => { now: number }; + ping: () => Promise<{ now: number }>; + terminate: () => Promise; + exec: (opts: ExecuteCodeOptions) => Promise; + realpath: (path: string) => Promise; } diff --git a/src/packages/nats/project-api/terminal.ts b/src/packages/nats/project-api/terminal.ts new file mode 100644 index 0000000000..fb6978e7c6 --- /dev/null +++ b/src/packages/nats/project-api/terminal.ts @@ -0,0 +1,13 @@ +export const terminal = { + create: true, + restart: true, + command: true, + write: true, +}; + +export interface Terminal { + create: (params) => Promise<{ subject: string }>; + restart: ({ path }) => Promise; + command: ({ path, cmd, ...args }) => Promise; + write: ({ data, path }) => Promise; +} diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index a9f827acd6..e9752563f4 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -5,3 +5,14 @@ import generateVouchers from "@cocalc/util/vouchers"; export function randomId() { return generateVouchers({ count: 1, length: 10 })[0]; } + +export function handleErrorMessage(mesg) { + if (mesg?.error) { + if (mesg.error.startsWith("Error: ")) { + throw Error(mesg.error.slice("Error: ".length)); + } else { + throw Error(mesg.error); + } + } + return mesg; +} diff --git a/src/packages/project/browser-websocket/exec-code.ts b/src/packages/project/browser-websocket/exec-code.ts index eb111a7782..01f55c64c9 100644 --- a/src/packages/project/browser-websocket/exec-code.ts +++ b/src/packages/project/browser-websocket/exec-code.ts @@ -12,12 +12,7 @@ import type { ExecuteCodeOutput, } from "@cocalc/util/types/execute-code"; -interface Options extends ExecuteCodeOptions { - compute_server_id?: number; - filesystem?: boolean; -} - -export default async function execCode(opts: Options): Promise { +export default async function execCode(opts: ExecuteCodeOptions): Promise { if (opts.compute_server_id) { if (opts.filesystem) { return await handleComputeServerFilesystemExec(opts); diff --git a/src/packages/project/exec_shell_code.ts b/src/packages/project/exec_shell_code.ts index 02b0f76609..bbaef7b0f0 100644 --- a/src/packages/project/exec_shell_code.ts +++ b/src/packages/project/exec_shell_code.ts @@ -11,7 +11,7 @@ import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; import * as message from "@cocalc/util/message"; import { getLogger } from "./logger"; import execCode from "@cocalc/project/browser-websocket/exec-code"; -import { ExecuteCodeOutput } from "@cocalc/util/types/execute-code"; +import type { ExecuteCodeOutput } from "@cocalc/util/types/execute-code"; const { debug: D } = getLogger("exec_shell_code"); diff --git a/src/packages/project/nats/api.ts b/src/packages/project/nats/api.ts deleted file mode 100644 index 535e855ccf..0000000000 --- a/src/packages/project/nats/api.ts +++ /dev/null @@ -1,112 +0,0 @@ -/* -How to do development (so in a dev project doing cc-in-cc dev). - -0. From the browser, terminate this api server running in the project already, if any - - await cc.client.nats_client.project({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e', endpoint:"terminate"}) - -1. Open a terminal in the project itself, which sets up the required environment variables, e.g., - - - COCALC_NATS_JWT -- this has the valid JWT issued to grant the project rights to use nats - - COCALC_PROJECT_ID - -You can type the following into the miniterminal in a project and copy the output into a terminal here to -setup the same environment and make starting this server act like this part of a project. - - export | grep -E "COCALC|HOME" - -2. Do this: - - echo 'require("@cocalc/project/nats/api").init()' | DEBUG=cocalc:* DEBUG_CONSOLE=yes node - -or just run node and paste - - require("@cocalc/project/nats/api").init() - -if you want to easily be able to grab some state, e.g., global.x = {...} in some code. - -5. Use the browser to see the project is on nats and works: - - await cc.client.nats_client.project({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e', endpoint:"exec", params:{command:'echo $COCALC_PROJECT_ID'}}) - -*/ - -import { getLogger } from "@cocalc/project/logger"; -import { JSONCodec } from "nats"; -import { project_id } from "@cocalc/project/data"; -import { handleExecShellCode } from "@cocalc/project/exec_shell_code"; -import { realpath } from "@cocalc/project/browser-websocket/realpath"; -import getConnection from "./connection"; - -const logger = getLogger("project:nats"); - -const jc = JSONCodec(); - -export async function init() { - const nc = await getConnection(); - const subject = `project.${project_id}.api.>`; - logger.debug(`initAPI -- NATS project subject '${subject}'`); - const sub = nc.subscribe(subject); - for await (const mesg of sub) { - const request = jc.decode(mesg.data) ?? {} as any; - if (request.endpoint == "terminate") { - mesg.respond(jc.encode({ status: "terminate" })); - // @ts-ignore - sub.close(); - return; - } - handleRequest(request, mesg, nc); - } -} - -async function handleRequest(request, mesg, nc) { - const segments = mesg.subject.split("."); - const group = segments[3]; // 'owner', 'collaborator', etc. - const account_id = segments[4]; - return await handleApiRequest({ request, mesg, group, account_id, nc }); -} - -async function handleApiRequest({ request, mesg, group, account_id, nc }) { - let resp; - try { - const { endpoint, params } = request; - logger.debug("handling project request:", { - endpoint, - params, - group, - account_id, - }); - resp = await getResponse({ endpoint, params, nc }); - } catch (err) { - resp = { error: `${err}` }; - } - logger.debug("responding with ", resp); - mesg.respond(jc.encode(resp)); -} - -import { - createTerminal, - restartTerminal, - terminalCommand, - writeToTerminal, -} from "./terminal"; -async function getResponse({ endpoint, params, nc }) { - switch (endpoint) { - case "ping": - return { pong: Date.now() }; - case "realpath": - return realpath(params.path); - case "exec": - return await handleExecShellCode(params); - case "create-terminal": - return await createTerminal({ params, nc }); - case "restart-terminal": - return await restartTerminal(params); - case "terminal-command": - return await terminalCommand(params); - case "write-to-terminal": - return await writeToTerminal(params); - default: - throw Error(`unknown endpoint '${endpoint}'`); - } -} diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index e1d23e247d..c694b29167 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -36,25 +36,31 @@ if you want to easily be able to grab some state, e.g., global.x = {...} in some import { JSONCodec } from "nats"; import getLogger from "@cocalc/backend/logger"; import { type ProjectApi } from "@cocalc/nats/project-api"; -import { getConnection } from "@cocalc/backend/nats"; +import getConnection from "@cocalc/project/nats/connection"; import { project_id } from "@cocalc/project/data"; const logger = getLogger("project:nats:api"); - const jc = JSONCodec(); export async function init() { const subject = `project.${project_id}.api`; - logger.debug(`initAPI -- subject='${subject}', options=`, { - queue: "0", - }); + logger.debug(`initAPI -- subject='${subject}'`); const nc = await getConnection(); - const sub = nc.subscribe(subject, { queue: "0" }); - for await (const mesg of sub) { + const subscription = nc.subscribe(subject); + logger.debug(`initAPI -- subscribed to subject='${subject}'`); + listen(subscription, subject); +} + +async function listen(subscription, subject) { + for await (const mesg of subscription) { const request = jc.decode(mesg.data) ?? ({} as any); - if (request.name == "terminate") { + // logger.debug("got message", request); + if (request.name == "system.terminate") { // special hook so admin can terminate handling. This is useful for development. + console.warn("TERMINATING listening on ", subject); + logger.debug("TERMINATING listening on ", subject); mesg.respond(jc.encode({ status: "terminating" })); + subscription.close(); return; } handleApiRequest(request, mesg); @@ -74,9 +80,11 @@ async function handleApiRequest(request, mesg) { } import * as system from "./system"; +import * as terminal from "./terminal"; export const projectApi: ProjectApi = { system, + terminal, }; async function getResponse({ name, args }) { diff --git a/src/packages/project/nats/api/system.ts b/src/packages/project/nats/api/system.ts index 19d6eef179..12d5de326d 100644 --- a/src/packages/project/nats/api/system.ts +++ b/src/packages/project/nats/api/system.ts @@ -1,3 +1,10 @@ -export function ping() { +export async function ping() { return { now: Date.now() }; } + +export async function terminate() {} + +import { handleExecShellCode } from "@cocalc/project/exec_shell_code"; +export { handleExecShellCode as exec }; + +export { realpath } from "@cocalc/project/browser-websocket/realpath"; diff --git a/src/packages/project/nats/api/terminal.ts b/src/packages/project/nats/api/terminal.ts new file mode 100644 index 0000000000..8fee037ef1 --- /dev/null +++ b/src/packages/project/nats/api/terminal.ts @@ -0,0 +1,8 @@ +import { + createTerminal as create, + restartTerminal as restart, + terminalCommand as command, + writeToTerminal as write, +} from "@cocalc/project/nats/terminal"; + +export { create, restart, command, write }; diff --git a/src/packages/project/nats/connection.ts b/src/packages/project/nats/connection.ts index e415440338..5c2d6bf66f 100644 --- a/src/packages/project/nats/connection.ts +++ b/src/packages/project/nats/connection.ts @@ -5,7 +5,7 @@ const logger = getLogger("project:nats:connection"); let nc: Awaited> | null = null; export default async function getConnection() { - if (nc == null) { + if (nc == null || (nc as any).protocol?.isClosed?.()) { logger.debug("initializing nats cocalc project connection"); if (!process.env.COCALC_NATS_JWT) { throw Error("environment variable COCALC_NATS_JWT *must* be set"); diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts index 2f98d24607..e3e71e55b4 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/nats/index.ts @@ -6,12 +6,13 @@ Start the NATS servers: */ import { getLogger } from "@cocalc/project/logger"; -const logger = getLogger("project:nats:index"); import { init as initAPI } from "./api"; import { init as initWebsocketApi } from "./browser-websocket-api"; +const logger = getLogger("project:nats:index"); + export default async function init() { logger.debug("starting NATS project servers"); - initAPI(); + await initAPI(); initWebsocketApi(); } diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index a8615f651a..ac07749fe4 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -16,6 +16,7 @@ import { JSONCodec } from "nats"; import { jetstreamManager } from "@nats-io/jetstream"; import { getLogger } from "@cocalc/project/logger"; import { readlink, realpath } from "node:fs/promises"; +import getConnection from "./connection"; const logger = getLogger("server:nats:terminal"); @@ -31,7 +32,7 @@ const jc = JSONCodec(); const sessions: { [name: string]: Session } = {}; export const createTerminal = reuseInFlight( - async ({ params, nc }: { params; nc }) => { + async (params) => { if (params == null) { throw Error("params must be specified"); } @@ -40,6 +41,7 @@ export const createTerminal = reuseInFlight( throw Error("path must be specified"); } if (sessions[path] == null) { + const nc = await getConnection(); sessions[path] = new Session({ path, options, nc }); await sessions[path].init(); } @@ -58,7 +60,6 @@ export async function writeToTerminal({ data, path }: { data; path }) { throw Error(`no terminal session '${path}'`); } await terminal.write(data); - return { success: true }; } export async function restartTerminal({ path }: { path }) { @@ -67,7 +68,6 @@ export async function restartTerminal({ path }: { path }) { throw Error(`no terminal session '${path}'`); } await terminal.restart(); - return { success: true }; } export async function terminalCommand({ path, cmd, ...args }) { diff --git a/src/packages/util/types/execute-code.ts b/src/packages/util/types/execute-code.ts index d6c3085250..3b72130e1b 100644 --- a/src/packages/util/types/execute-code.ts +++ b/src/packages/util/types/execute-code.ts @@ -48,6 +48,10 @@ export interface ExecuteCodeOptions { aggregate?: string | number; // if given, aggregates multiple calls with same sequence number into one -- see @cocalc/util/aggregate; typically make this a timestamp for compiling code (e.g., latex). verbose?: boolean; // default true -- impacts amount of logging async_call?: boolean; // default false -- if true, return right after the process started (to get the PID) or when it fails. + // for compute servers: + compute_server_id?: number; + // in the filesystem container of a compute server + filesystem?: boolean; } export interface ExecuteCodeOptionsAsyncGet { From 5a610b1d15a4c433ac73174b787159fa0842d4e7 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 30 Jan 2025 18:33:50 +0000 Subject: [PATCH 069/281] nats project api: add more system functions --- src/packages/nats/project-api/system.ts | 26 +++++++++++++++-- src/packages/project/client.ts | 3 +- src/packages/project/nats/api/system.ts | 38 +++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/packages/nats/project-api/system.ts b/src/packages/nats/project-api/system.ts index b66ca6c279..fe440c4881 100644 --- a/src/packages/nats/project-api/system.ts +++ b/src/packages/nats/project-api/system.ts @@ -2,17 +2,39 @@ import type { ExecuteCodeOutput, ExecuteCodeOptions, } from "@cocalc/util/types/execute-code"; +import type { DirectoryListingEntry } from "@cocalc/util/types"; export const system = { - ping: true, terminate: true, + + version: true, + + listing: true, + deleteFiles: true, + moveFiles: true, + renameFile: true, + + ping: true, exec: true, realpath: true, }; export interface System { - ping: () => Promise<{ now: number }>; terminate: () => Promise; + + version: () => Promise; + + listing: (opts: { + path: string; + hidden?: boolean; + }) => Promise; + deleteFiles: (opts: { paths: string[] }) => Promise; + moveFiles: (opts: { paths: string[]; dest: string }) => Promise; + renameFile: (opts: { src: string; dest: string }) => Promise; + + ping: () => Promise<{ now: number }>; + exec: (opts: ExecuteCodeOptions) => Promise; + realpath: (path: string) => Promise; } diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index 8bddb815d1..76f5b7ea0a 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -87,7 +87,8 @@ export function init() { export function getClient(): Client { if (client == null) { - throw Error("BUG: Client not initialized!"); + init(); + //throw Error("BUG: Client not initialized!"); } return client; } diff --git a/src/packages/project/nats/api/system.ts b/src/packages/project/nats/api/system.ts index 12d5de326d..1e073c7601 100644 --- a/src/packages/project/nats/api/system.ts +++ b/src/packages/project/nats/api/system.ts @@ -8,3 +8,41 @@ import { handleExecShellCode } from "@cocalc/project/exec_shell_code"; export { handleExecShellCode as exec }; export { realpath } from "@cocalc/project/browser-websocket/realpath"; + +import { version as versionNumber } from "@cocalc/util/smc-version"; +export async function version() { + return versionNumber; +} + +import getListing from "@cocalc/backend/get-listing"; +export async function listing({ path, hidden }) { + return await getListing(path, hidden); +} + +import { delete_files } from "@cocalc/backend/files/delete-files"; + +export async function deleteFiles({ paths }: { paths: string[] }) { + return await delete_files(paths); +} + +import { getClient } from "@cocalc/project/client"; +async function setDeleted(path) { + const client = getClient(); + await client.set_deleted(path); +} + +import { move_files } from "@cocalc/backend/files/move-files"; +export async function moveFiles({ + paths, + dest, +}: { + paths: string[]; + dest: string; +}) { + await move_files(paths, dest, setDeleted); +} + +import { rename_file } from "@cocalc/backend/files/rename-file"; +export async function renameFile({ src, dest }: { src: string; dest: string }) { + await rename_file(src, dest, setDeleted); +} From 1f701f5cc72f8d0d838a63f9098c1d1dc5e8aa07 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 30 Jan 2025 18:49:57 +0000 Subject: [PATCH 070/281] nats project api: add configuration --- src/packages/nats/package.json | 1 + src/packages/nats/project-api/system.ts | 12 +++++++++++ src/packages/nats/tsconfig.json | 2 +- src/packages/pnpm-lock.yaml | 3 +++ src/packages/project/browser-websocket/api.ts | 21 +++++++++++++++---- src/packages/project/nats/api/system.ts | 3 +++ 6 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 13a53ef247..5ea256b56d 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -23,6 +23,7 @@ "dependencies": { "@cocalc/nats": "workspace:*", "@cocalc/util": "workspace:*", + "@cocalc/comm": "workspace:*", "@nats-io/jetstream": "3.0.0-36", "@nats-io/kv": "3.0.0-30", "awaiting": "^3.0.0", diff --git a/src/packages/nats/project-api/system.ts b/src/packages/nats/project-api/system.ts index fe440c4881..11b727605f 100644 --- a/src/packages/nats/project-api/system.ts +++ b/src/packages/nats/project-api/system.ts @@ -3,6 +3,10 @@ import type { ExecuteCodeOptions, } from "@cocalc/util/types/execute-code"; import type { DirectoryListingEntry } from "@cocalc/util/types"; +import type { + Configuration, + ConfigurationAspect, +} from "@cocalc/comm/project-configuration"; export const system = { terminate: true, @@ -13,6 +17,9 @@ export const system = { deleteFiles: true, moveFiles: true, renameFile: true, + canonicalPaths: true, + + configuration: true, ping: true, exec: true, @@ -32,6 +39,11 @@ export interface System { moveFiles: (opts: { paths: string[]; dest: string }) => Promise; renameFile: (opts: { src: string; dest: string }) => Promise; + configuration: ( + aspect: ConfigurationAspect, + no_cache?, + ) => Promise; + ping: () => Promise<{ now: number }>; exec: (opts: ExecuteCodeOptions) => Promise; diff --git a/src/packages/nats/tsconfig.json b/src/packages/nats/tsconfig.json index 6cdb913e19..a0c8debf1a 100644 --- a/src/packages/nats/tsconfig.json +++ b/src/packages/nats/tsconfig.json @@ -5,5 +5,5 @@ "outDir": "dist" }, "exclude": ["node_modules", "dist", "test"], - "references": [{ "path": "../util" }] + "references": [{ "path": "../util" }, { "path": "../comm" }] } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 57dacd66e2..3e42d11f81 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1051,6 +1051,9 @@ importers: nats: dependencies: + '@cocalc/comm': + specifier: workspace:* + version: link:../comm '@cocalc/nats': specifier: workspace:* version: 'link:' diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts index ee26789cfd..dd844892cf 100644 --- a/src/packages/project/browser-websocket/api.ts +++ b/src/packages/project/browser-websocket/api.ts @@ -148,24 +148,35 @@ export async function handleApiCall({ throw Error("opts must not be null"); } return await execCode(data.opts); + case "realpath": + return realpath(data.path); + + // todo: why? case "query": return await query(client, data.opts); + // todo: why? case "eval_code": return await eval_code(data.code); + case "terminal": return await terminal(primus, data.path, data.options); - case "lean": - return await lean(client, primus, log, data.opts); + + case "jupyter_strip_notebook": return await jupyter_strip_notebook(data.ipynb_path); case "jupyter_nbconvert": return await jupyter_nbconvert(data.opts); case "jupyter_run_notebook": return await jupyter_run_notebook(log, data.opts); + + case "lean": + return await lean(client, primus, log, data.opts); case "lean_channel": return await lean_channel(client, primus, log, data.path); + case "x11_channel": return await x11_channel(client, primus, log, data.path, data.display); + case "synctable_channel": return await synctable_channel( client, @@ -178,17 +189,19 @@ export async function handleApiCall({ return await syncdoc_call(data.path, data.mesg); case "symmetric_channel": return await browser_symmetric_channel(client, primus, log, data.name); - case "realpath": - return realpath(data.path); + + case "project_info": return await project_info_ws(primus, log); case "compute_filesystem_cache": return await computeFilesystemCache(data.opts); case "sync_fs": return await handleSyncFsApiCall(data.opts); + case "compute_server_sync_register": // register filesystem container return await handleComputeServerSyncRegister(data.opts, spark); + case "compute_server_compute_register": // register compute container return await handleComputeServerComputeRegister(data.opts, spark); diff --git a/src/packages/project/nats/api/system.ts b/src/packages/project/nats/api/system.ts index 1e073c7601..9c1e29d04b 100644 --- a/src/packages/project/nats/api/system.ts +++ b/src/packages/project/nats/api/system.ts @@ -46,3 +46,6 @@ import { rename_file } from "@cocalc/backend/files/rename-file"; export async function renameFile({ src, dest }: { src: string; dest: string }) { await rename_file(src, dest, setDeleted); } + +import { get_configuration } from "@cocalc/project/configuration"; +export { get_configuration as configuration }; From c2c81b20362422ad88952ad900c5fe073689c683 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 30 Jan 2025 20:31:41 +0000 Subject: [PATCH 071/281] nats project api -- implemented basically everything but sync support on backend --- src/packages/comm/websocket/types.ts | 2 +- src/packages/frontend/jupyter/nbgrader/api.ts | 2 +- .../frontend/jupyter/nbgrader/autograde.ts | 2 +- src/packages/jupyter/kernel/kernel.ts | 2 +- src/packages/jupyter/nbgrader/jupyter-run.ts | 18 ++++++---- src/packages/nats/project-api/editor.ts | 30 ++++++++++++++++ src/packages/nats/project-api/index.ts | 8 ++++- src/packages/nats/project-api/sync.ts | 12 +++++++ src/packages/project/browser-websocket/api.ts | 19 +++++----- src/packages/project/formatters/index.ts | 36 ++++++++++++------- src/packages/project/jupyter/convert/index.ts | 2 +- src/packages/project/nats/api/editor.ts | 7 ++++ src/packages/project/nats/api/index.ts | 4 +++ src/packages/project/nats/api/sync.ts | 3 ++ src/packages/project/package.json | 4 +-- .../jupyter/nbgrader-types.ts} | 5 ++- .../nbconvert.ts => util/jupyter/types.ts} | 0 17 files changed, 115 insertions(+), 41 deletions(-) create mode 100644 src/packages/nats/project-api/editor.ts create mode 100644 src/packages/nats/project-api/sync.ts create mode 100644 src/packages/project/nats/api/editor.ts create mode 100644 src/packages/project/nats/api/sync.ts rename src/packages/{jupyter/nbgrader/types.ts => util/jupyter/nbgrader-types.ts} (97%) rename src/packages/{jupyter/types/nbconvert.ts => util/jupyter/types.ts} (100%) diff --git a/src/packages/comm/websocket/types.ts b/src/packages/comm/websocket/types.ts index 43c33bfe0b..4264656c5a 100644 --- a/src/packages/comm/websocket/types.ts +++ b/src/packages/comm/websocket/types.ts @@ -11,7 +11,7 @@ between the frontend app and the project. import type { NBGraderAPIOptions, RunNotebookOptions, -} from "@cocalc/jupyter/nbgrader/types"; +} from "@cocalc/util/jupyter/nbgrader-types"; import type { Channel } from "@cocalc/sync/client/types"; import type { Options } from "@cocalc/util/code-formatter"; export type { Channel }; diff --git a/src/packages/frontend/jupyter/nbgrader/api.ts b/src/packages/frontend/jupyter/nbgrader/api.ts index 3044ff3f56..ab15702425 100644 --- a/src/packages/frontend/jupyter/nbgrader/api.ts +++ b/src/packages/frontend/jupyter/nbgrader/api.ts @@ -10,7 +10,7 @@ import type { NBGraderAPIOptions, NBGraderAPIResponse, RunNotebookOptions, -} from "@cocalc/jupyter/nbgrader/types"; +} from "@cocalc/util/jupyter/nbgrader-types"; export type { NBGraderAPIOptions, RunNotebookOptions }; export async function nbgrader( diff --git a/src/packages/frontend/jupyter/nbgrader/autograde.ts b/src/packages/frontend/jupyter/nbgrader/autograde.ts index 005504c1f6..ed884f2f49 100644 --- a/src/packages/frontend/jupyter/nbgrader/autograde.ts +++ b/src/packages/frontend/jupyter/nbgrader/autograde.ts @@ -23,7 +23,7 @@ that get used in testing. import { copy, is_array, startswith } from "@cocalc/util/misc"; import { state_to_value } from "./cell-types"; -import type { JupyterNotebook, Cell } from "@cocalc/jupyter/nbgrader/types"; +import type { JupyterNotebook, Cell } from "@cocalc/util/jupyter/nbgrader-types"; export function create_autograde_ipynb( instructor_ipynb: string, diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index 77cffb7327..c88333417c 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -76,7 +76,7 @@ import type { KernelParams } from "@cocalc/jupyter/types/kernel"; import { redux_name } from "@cocalc/util/redux/name"; import { redux } from "@cocalc/jupyter/redux/app"; import { VERSION } from "@cocalc/jupyter/kernel/version"; -import type { NbconvertParams } from "@cocalc/jupyter/types/nbconvert"; +import type { NbconvertParams } from "@cocalc/util/jupyter/types"; import type { Client } from "@cocalc/sync/client/types"; import { getLogger } from "@cocalc/backend/logger"; import { base64ToBuffer } from "@cocalc/util/base64"; diff --git a/src/packages/jupyter/nbgrader/jupyter-run.ts b/src/packages/jupyter/nbgrader/jupyter-run.ts index b9caae3b20..ebe6093f4f 100644 --- a/src/packages/jupyter/nbgrader/jupyter-run.ts +++ b/src/packages/jupyter/nbgrader/jupyter-run.ts @@ -3,12 +3,17 @@ * License: MS-RSL – see LICENSE.md for details */ -import type { RunNotebookOptions } from "@cocalc/jupyter/nbgrader/types"; -import type { JupyterNotebook } from "@cocalc/jupyter/nbgrader/types"; +import type { + JupyterNotebook, + RunNotebookOptions, +} from "@cocalc/util/jupyter/nbgrader-types"; import type { JupyterKernelInterface as JupyterKernel } from "@cocalc/jupyter/types/project-interface"; import { is_object, len, uuid, trunc_middle } from "@cocalc/util/misc"; import { retry_until_success } from "@cocalc/util/async-utils"; import { kernel } from "@cocalc/jupyter/kernel"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("jupyter:nbgrader:jupyter-run"); // For tracking limits during the run: export interface Limits { @@ -26,8 +31,7 @@ function global_timeout_exceeded(limits: Limits): boolean { } export async function jupyter_run_notebook( - logger, - opts: RunNotebookOptions + opts: RunNotebookOptions, ): Promise { const log = (...args) => { logger.debug("jupyter_run_notebook", ...args); @@ -129,7 +133,7 @@ export async function jupyter_run_notebook( export async function run_cell( jupyter: JupyterKernel, limits: Limits, - cell + cell, ): Promise { if (jupyter == null) { throw Error("jupyter must be defined"); @@ -140,8 +144,8 @@ export async function run_cell( // for each cell in the rest of the notebook. throw Error( `Total time limit (=${Math.round( - limits.timeout_ms / 1000 - )} seconds) exceeded` + limits.timeout_ms / 1000, + )} seconds) exceeded`, ); } diff --git a/src/packages/nats/project-api/editor.ts b/src/packages/nats/project-api/editor.ts new file mode 100644 index 0000000000..8571fcf599 --- /dev/null +++ b/src/packages/nats/project-api/editor.ts @@ -0,0 +1,30 @@ +import type { NbconvertParams } from "@cocalc/util/jupyter/types"; +import type { RunNotebookOptions } from "@cocalc/util/jupyter/nbgrader-types"; +import type { Options as FormatterOptions } from "@cocalc/util/code-formatter"; + +export const editor = { + jupyterStripNotebook: true, + jupyterNbconvert: true, + jupyterRunNotebook: true, + + formatter: true, + formatterString: true, +}; + +export interface Editor { + jupyterStripNotebook: (path_ipynb: string) => Promise; + jupyterNbconvert: (opts: NbconvertParams) => Promise; + jupyterRunNotebook: (opts: RunNotebookOptions) => Promise; + + // returns a patch to transform doc into formatted form. + formatter: (opts: { + path: string; + options: FormatterOptions; + }) => Promise; + + formatterString: (opts: { + str: string; + options: FormatterOptions; + path?: string; // only used for CLANG + }) => Promise; +} diff --git a/src/packages/nats/project-api/index.ts b/src/packages/nats/project-api/index.ts index ffaf939955..9e0f528b30 100644 --- a/src/packages/nats/project-api/index.ts +++ b/src/packages/nats/project-api/index.ts @@ -1,15 +1,21 @@ import { type System, system } from "./system"; import { type Terminal, terminal } from "./terminal"; -import { handleErrorMessage} from "@cocalc/nats/util"; +import { type Editor, editor } from "./editor"; +import { type Sync, sync } from "./sync"; +import { handleErrorMessage } from "@cocalc/nats/util"; export interface ProjectApi { system: System; terminal: Terminal; + editor: Editor; + sync: Sync; } const ProjectApiStructure = { system, terminal, + editor, + sync, } as const; export function initProjectApi(callProjectApi): ProjectApi { diff --git a/src/packages/nats/project-api/sync.ts b/src/packages/nats/project-api/sync.ts new file mode 100644 index 0000000000..8acbac3716 --- /dev/null +++ b/src/packages/nats/project-api/sync.ts @@ -0,0 +1,12 @@ +export const sync = { + close: true, + // projectInfo: true, + + // x11: true, + // synctableChannel: true, + // symmetricChannel: true, +}; + +export interface Sync { + close: (path: string) => Promise; +} diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts index dd844892cf..67f2149e3d 100644 --- a/src/packages/project/browser-websocket/api.ts +++ b/src/packages/project/browser-websocket/api.ts @@ -139,10 +139,10 @@ export async function handleApiCall({ return await get_configuration(data.aspect, data.no_cache); case "prettier": // deprecated case "formatter": - return await run_formatter(client, data.path, data.options, log); + return await run_formatter(data); case "prettier_string": // deprecated case "formatter_string": - return await run_formatter_string(data.path, data.str, data.options, log); + return await run_formatter_string(data); case "exec": if (data.opts == null) { throw Error("opts must not be null"); @@ -161,13 +161,12 @@ export async function handleApiCall({ case "terminal": return await terminal(primus, data.path, data.options); - case "jupyter_strip_notebook": return await jupyter_strip_notebook(data.ipynb_path); case "jupyter_nbconvert": return await jupyter_nbconvert(data.opts); case "jupyter_run_notebook": - return await jupyter_run_notebook(log, data.opts); + return await jupyter_run_notebook(data.opts); case "lean": return await lean(client, primus, log, data.opts); @@ -176,7 +175,7 @@ export async function handleApiCall({ case "x11_channel": return await x11_channel(client, primus, log, data.path, data.display); - + case "synctable_channel": return await synctable_channel( client, @@ -189,19 +188,21 @@ export async function handleApiCall({ return await syncdoc_call(data.path, data.mesg); case "symmetric_channel": return await browser_symmetric_channel(client, primus, log, data.name); - - + case "project_info": return await project_info_ws(primus, log); + + // compute server + case "compute_filesystem_cache": return await computeFilesystemCache(data.opts); case "sync_fs": return await handleSyncFsApiCall(data.opts); - + case "compute_server_sync_register": // register filesystem container return await handleComputeServerSyncRegister(data.opts, spark); - + case "compute_server_compute_register": // register compute container return await handleComputeServerComputeRegister(data.opts, spark); diff --git a/src/packages/project/formatters/index.ts b/src/packages/project/formatters/index.ts index 78ee49b361..d255325e76 100644 --- a/src/packages/project/formatters/index.ts +++ b/src/packages/project/formatters/index.ts @@ -37,13 +37,19 @@ import type { Options, } from "@cocalc/util/code-formatter"; export type { Config, Options, FormatterSyntax }; +import { getLogger } from "@cocalc/backend/logger"; +import { getClient } from "@cocalc/project/client"; -export async function run_formatter( - client: any, - path: string, - options: Options, - logger: any, -): Promise { +const logger = getLogger("project:formatters"); + +export async function run_formatter({ + path, + options, +}: { + path: string; + options: Options; +}): Promise { + const client = getClient(); // What we do is edit the syncstring with the given path to be "prettier" if possible... const syncstring = client.syncdoc({ path }); if (syncstring == null || syncstring.get_state() == "closed") { @@ -63,7 +69,7 @@ export async function run_formatter( [input, math] = remove_math(math_escape(input)); } try { - formatted = await run_formatter_string(path, input, options, logger); + formatted = await run_formatter_string({ path, str: input, options }); } catch (err) { logger.debug(`run_formatter error: ${err.message}`); return { status: "error", phase: "format", error: err.message }; @@ -78,13 +84,17 @@ export async function run_formatter( return { status: "ok", patch }; } -export async function run_formatter_string( - path: string | undefined, - input: string, - options: Options, - logger: any, -): Promise { +export async function run_formatter_string({ + options, + str, + path, +}: { + str: string; + options: Options; + path?: string; // only used for CLANG +}): Promise { let formatted; + const input = str; logger.debug(`run_formatter options.parser: "${options.parser}"`); switch (options.parser) { case "latex": diff --git a/src/packages/project/jupyter/convert/index.ts b/src/packages/project/jupyter/convert/index.ts index a8e65a019d..e9fdbd522c 100644 --- a/src/packages/project/jupyter/convert/index.ts +++ b/src/packages/project/jupyter/convert/index.ts @@ -14,7 +14,7 @@ import { parseSource, parseTo } from "./util"; import { join } from "path"; import { getLogger } from "@cocalc/project/logger"; import { sanitize_nbconvert_path } from "@cocalc/util/sanitize-nbconvert"; -import type { NbconvertParams } from "@cocalc/jupyter/types/nbconvert"; +import type { NbconvertParams } from "@cocalc/util/jupyter/types"; const log = getLogger("jupyter-nbconvert"); diff --git a/src/packages/project/nats/api/editor.ts b/src/packages/project/nats/api/editor.ts new file mode 100644 index 0000000000..bda781480f --- /dev/null +++ b/src/packages/project/nats/api/editor.ts @@ -0,0 +1,7 @@ +export { jupyter_strip_notebook as jupyterStripNotebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; +export { jupyter_run_notebook as jupyterRunNotebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; +export { nbconvert as jupyterNbconvert } from "../../jupyter/convert"; +export { + run_formatter as formatter, + run_formatter_string as formatterString, +} from "../../formatters"; diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index c694b29167..a1cd5b39df 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -81,10 +81,14 @@ async function handleApiRequest(request, mesg) { import * as system from "./system"; import * as terminal from "./terminal"; +import * as editor from "./editor"; +import * as sync from "./sync"; export const projectApi: ProjectApi = { system, terminal, + editor, + sync, }; async function getResponse({ name, args }) { diff --git a/src/packages/project/nats/api/sync.ts b/src/packages/project/nats/api/sync.ts new file mode 100644 index 0000000000..2690f40249 --- /dev/null +++ b/src/packages/project/nats/api/sync.ts @@ -0,0 +1,3 @@ +export async function close(path: string) { + console.log("TODO: close path", { path }); +} diff --git a/src/packages/project/package.json b/src/packages/project/package.json index aa7d47a5c8..6cb0cc4eeb 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -88,9 +88,7 @@ "clean": "rm -rf dist" }, "author": "SageMath, Inc.", - "contributors": [ - "William Stein " - ], + "contributors": ["William Stein "], "license": "SEE LICENSE.md", "bugs": { "url": "https://github.com/sagemathinc/cocalc/issues" diff --git a/src/packages/jupyter/nbgrader/types.ts b/src/packages/util/jupyter/nbgrader-types.ts similarity index 97% rename from src/packages/jupyter/nbgrader/types.ts rename to src/packages/util/jupyter/nbgrader-types.ts index 84ada98b17..3b76f4ec1f 100644 --- a/src/packages/jupyter/nbgrader/types.ts +++ b/src/packages/util/jupyter/nbgrader-types.ts @@ -1,4 +1,3 @@ - export interface NBGraderAPIOptions { // Project will try to evaluate/autograde for this many milliseconds; // if time is exceeded, all additional problems fail and what we graded @@ -36,7 +35,6 @@ export interface NBGraderAPIResponse { ids: string[]; } - export interface RunNotebookLimits { max_output?: number; // any output that pushes the total length beyond this many characters is ignored. max_output_per_cell?: number; // any output that pushes a single cell's output beyond this many characters is ignored. @@ -45,13 +43,14 @@ export interface RunNotebookLimits { } export interface RunNotebookOptions { + // where to run it path: string; + // contents of the ipynb file (NOT THE PATH) ipynb: string; nbgrader?: boolean; // if true, only record outputs for nbgrader autograder cells (all cells are run, but only these get output) limits?: RunNotebookLimits; } - // Enough description of what a Jupyter notebook is for our purposes here. export interface Cell { cell_type: "code" | "markdown" | "raw"; diff --git a/src/packages/jupyter/types/nbconvert.ts b/src/packages/util/jupyter/types.ts similarity index 100% rename from src/packages/jupyter/types/nbconvert.ts rename to src/packages/util/jupyter/types.ts From 41d45083835372c02551486c82f2caaf37ba5a42 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 31 Jan 2025 00:17:28 +0000 Subject: [PATCH 072/281] nats projects: cleaning up naming (via refactor) and supporting compute servers from the start --- src/packages/frontend/client/nats.ts | 92 +++++++------------ .../terminal-editor/connected-terminal.ts | 1 + .../nats-terminal-connection.ts | 32 +++++-- .../frontend/project/websocket/api.ts | 47 +++++----- src/packages/nats/names.ts | 56 +++++++++++ src/packages/nats/util.ts | 9 +- src/packages/project/data.ts | 1 + src/packages/project/nats/api/index.ts | 4 +- .../project/nats/browser-websocket-api.ts | 10 +- src/packages/project/nats/names.ts | 10 ++ src/packages/project/nats/terminal.ts | 13 ++- 11 files changed, 166 insertions(+), 109 deletions(-) create mode 100644 src/packages/nats/names.ts create mode 100644 src/packages/project/nats/names.ts diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 701588d99f..e6bf5b58c9 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -3,16 +3,17 @@ import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import type { WebappClient } from "./client"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { join } from "path"; -import { redux } from "../app-framework"; import * as jetstream from "@nats-io/jetstream"; import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; -import { randomId } from "@cocalc/nats/util"; +import { randomId } from "@cocalc/nats/names"; +import { projectSubject } from "@cocalc/nats/names"; import { parse_query } from "@cocalc/sync/table/util"; import { sha1 } from "@cocalc/util/misc"; import { keys } from "lodash"; import { type HubApi, initHubApi } from "@cocalc/nats/hub-api"; import { type ProjectApi, initProjectApi } from "@cocalc/nats/project-api"; import { getPrimusConnection } from "@cocalc/nats/primus"; +import { isValidUUID } from "@cocalc/util/misc"; export class NatsClient { /*private*/ client: WebappClient; @@ -59,9 +60,10 @@ export class NatsClient { return this.nc; }); + // deprecated! projectWebsocketApi = async ({ project_id, mesg, timeout = 5000 }) => { const nc = await this.getConnection(); - const subject = `project.${project_id}.browser-api`; + const subject = `${projectSubject({ project_id })}.browser-api`; const resp = await nc.request(subject, this.jc.encode(mesg), { timeout, }); @@ -95,14 +97,20 @@ export class NatsClient { // Returns api for RPC calls to the project with typescript support! projectApi = ({ project_id, + compute_server_id = 0, timeout, }: { project_id: string; + compute_server_id?: number; timeout?: number; }): ProjectApi => { + if (!isValidUUID(project_id)) { + throw Error(`project_id = '${project_id}' must be a valid uuid`); + } const callProjectApi = async ({ name, args }) => { return await this.callProject({ project_id, + compute_server_id, timeout, service: "api", name, @@ -115,26 +123,36 @@ export class NatsClient { private callProject = async ({ service = "api", project_id, + compute_server_id, name, args = [], timeout = 5000, }: { service?: string; project_id: string; + compute_server_id: number; name: string; args: any[]; timeout?: number; }) => { const nc = await this.getConnection(); - const subject = `project.${project_id}.${service}`; - const resp = await nc.request( - subject, - this.jc.encode({ - name, - args, - }), - { timeout }, - ); + const subject = `${projectSubject({ project_id, compute_server_id })}.${service}`; + const mesg = this.jc.encode({ + name, + args, + }); + let resp; + try { + resp = await nc.request(subject, mesg, { timeout }); + } catch (err) { + if (err.code == "PERMISSIONS_VIOLATION") { + // request update of our credentials to include this project, then try again + await this.hub.system.addProjectPermission({ project_id }); + resp = await nc.request(subject, mesg, { timeout }); + } else { + throw err; + } + } return this.jc.decode(resp.data); }; @@ -144,45 +162,6 @@ export class NatsClient { return this.sc.decode(resp.data); }; - project = async ({ - project_id, - endpoint, - params, - }: { - project_id: string; - endpoint: string; - params?: object; - }) => { - const c = await this.getConnection(); - const group = redux.getProjectsStore().get_my_group(project_id); - if (!group) { - // todo...? - throw Error(`group not yet known for '${project_id}'`); - } - const subject = `project.${project_id}.api.${group}.${this.client.account_id}`; - const resp = await c.request( - subject, - this.jc.encode({ - endpoint, - params, - }), - ); - const x = this.jc.decode(resp.data) as any; - if (x?.error) { - throw Error(x.error); - } - return x; - }; - - // for debugging -- listen to and display all messages on a subject - subscribe = async (subject: string) => { - const nc = await this.getConnection(); - const sub = nc.subscribe(subject); - for await (const mesg of sub) { - console.log(this.jc.decode(mesg.data)); - } - }; - consumer = async (stream: string) => { const js = jetstream.jetstream(await this.getConnection()); return await js.consumers.get(stream); @@ -242,17 +221,10 @@ export class NatsClient { return await this.synctable(query, { atomic: true }); }; - // createSocket = async (subjects: { listen: string; send: string }) => { - // return new Socket({ - // ...subjects, - // nc: await this.getConnection(), - // jc: this.jc, - // }); - // }; - + // DEPRECATED primus = async (project_id: string) => { return getPrimusConnection({ - subject: `project.${project_id}.primus`, + subject: `${projectSubject({ project_id, compute_server_id: 0 })}.primus`, env: await this.getEnv(), role: "client", id: this.sessionId, diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index c82ed2eac8..d5f72f28ed 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -284,6 +284,7 @@ export class Terminal { terminalResize: this.terminal_resize, openPaths: this.open_paths, closePaths: this.close_paths, + compute_server_id: await this.getComputeServerId(), }); this.conn = conn as any; conn.on("close", this.connect); diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index aefef536da..667295dfd7 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -2,14 +2,16 @@ import { webapp_client } from "@cocalc/frontend/webapp-client"; import { EventEmitter } from "events"; import { JSONCodec } from "nats.ws"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { sha1, uuid } from "@cocalc/util/misc"; +import { uuid } from "@cocalc/util/misc"; import { delay } from "awaiting"; +import { projectStreamName, projectSubject } from "@cocalc/nats/names"; const jc = JSONCodec(); const client = uuid(); export class NatsTerminalConnection extends EventEmitter { private project_id: string; + private compute_server_id: number; private path: string; private subject: string; private cmd_subject: string; @@ -25,6 +27,7 @@ export class NatsTerminalConnection extends EventEmitter { constructor({ project_id, + compute_server_id, path, keep, terminalResize, @@ -32,6 +35,7 @@ export class NatsTerminalConnection extends EventEmitter { closePaths, }: { project_id: string; + compute_server_id: number; path: string; keep?: number; terminalResize; @@ -40,15 +44,25 @@ export class NatsTerminalConnection extends EventEmitter { }) { super(); this.project_id = project_id; + this.compute_server_id = compute_server_id; this.path = path; this.terminalResize = terminalResize; this.keep = keep; this.openPaths = openPaths; this.closePaths = closePaths; this.project = webapp_client.nats_client.projectApi({ project_id }); - // TODO: move to @cocalc/nats (?) so guaranteed in sync with project - this.subject = `project.${project_id}.terminal.${sha1(path)}`; - this.cmd_subject = `project.${project_id}.terminal-cmd.${sha1(path)}`; + this.subject = projectSubject({ + project_id, + compute_server_id, + service: "terminal", + path, + }); + this.cmd_subject = projectSubject({ + project_id, + compute_server_id, + service: "terminal-cmd", + path, + }); } write = async (data) => { @@ -107,16 +121,20 @@ export class NatsTerminalConnection extends EventEmitter { private getConsumer = async () => { // TODO: idempotent, but move to project const { nats_client } = webapp_client; - const stream = `project-${this.project_id}-terminal`; + const streamName = projectStreamName({ + project_id: this.project_id, + compute_server_id: this.compute_server_id, + service: "terminal", + }); const nc = await nats_client.getConnection(); const js = nats_client.jetstream.jetstream(nc); // consumer doesn't exist, so setup everything. const jsm = await nats_client.jetstream.jetstreamManager(nc); // making an ephemeral consumer for just one subject (e.g., this terminal frame) - const { name } = await jsm.consumers.add(stream, { + const { name } = await jsm.consumers.add(streamName, { filter_subject: this.subject, }); - return await js.consumers.get(stream, name); + return await js.consumers.get(streamName, name); }; init = async () => { diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 0eca13304b..6d3ec59ffe 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -30,11 +30,13 @@ import type { } from "@cocalc/comm/websocket/types"; import call from "@cocalc/sync/client/call"; import { webapp_client } from "@cocalc/frontend/webapp-client"; +import { type ProjectApi } from "@cocalc/nats/project-api"; export class API { private conn; private project_id: string; private cachedVersion?: number; + private apiCache: { [key: string]: ProjectApi } = {}; constructor(conn, project_id: string) { this.conn = conn; @@ -45,6 +47,24 @@ export class API { }); } + private getApi = ({ + compute_server_id = 0, + timeout = 5000, + }: { + compute_server_id?: number; + timeout?: number; + }) => { + const key = `${compute_server_id}-${timeout}`; + if (this.apiCache[key] == null) { + this.apiCache[key] = webapp_client.nats_client.projectApi({ + project_id: this.project_id, + compute_server_id, + timeout, + }); + } + return this.apiCache[key]!; + }; + private primusCall = async (mesg: Mesg, timeout: number) => { return await call(this.conn, mesg, timeout); }; @@ -79,34 +99,17 @@ export class API { } }; - version = async (): Promise => { - // version can never change (except when you restart the project!), so its safe to cache - if (this.cachedVersion != null) { - return this.cachedVersion; - } - try { - this.cachedVersion = await this.call({ cmd: "version" }, 15000); - } catch (err) { - if (err.message.includes('command "version" not implemented')) { - this.cachedVersion = 0; - } else { - throw err; - } - } - if (this.cachedVersion == null) { - return 0; - } - return this.cachedVersion; + version = async (compute_server_id?: number): Promise => { + const api = this.getApi({ compute_server_id }); + return await api.system.version(); }; delete_files = async ( paths: string[], compute_server_id?: number, ): Promise => { - return await this.call( - { cmd: "delete_files", paths, compute_server_id }, - 60000, - ); + const api = this.getApi({ compute_server_id, timeout: 60000 }); + return await api.system.deleteFiles({ paths }); }; // Move the given paths to the dest. The folder dest must exist diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts new file mode 100644 index 0000000000..7bb637705b --- /dev/null +++ b/src/packages/nats/names.ts @@ -0,0 +1,56 @@ +import { sha1 } from "@cocalc/util/misc"; +import generateVouchers from "@cocalc/util/vouchers"; + +// nice alphanumeric string that can be used as nats subject, and very +// unlikely to randomly collide with another browser tab from this account. +export function randomId() { + return generateVouchers({ count: 1, length: 10 })[0]; +} + +// project-{project_id}-{compute_server_id}[.-service][.-sha1(path)] + +export function projectSubject({ + project_id, + compute_server_id = 0, + // service = optional name of the microservice, e.g., 'api', 'terminal' + service, + // path = optional name of specific path for that microservice -- replaced by its sha1 + path, +}: { + project_id: string; + compute_server_id?: number; + service?: string; + path?: string; +}): string { + let subject = `project.${project_id}.${compute_server_id}`; + if (service) { + subject += "." + service; + if (path) { + subject += "." + sha1(path); + } + } + return subject; +} + +export function projectStreamName({ + project_id, + compute_server_id = 0, + // service = optional name of the microservice, e.g., 'api', 'terminal' + service, + // path = optional name of specific path for that microservice -- replaced by its sha1 + path, +}: { + project_id: string; + compute_server_id?: number; + service?: string; + path?: string; +}): string { + let streamName = `project-${project_id}-${compute_server_id}`; + if (service) { + streamName += "-" + service; + if (path) { + streamName += "-" + sha1(path); + } + } + return streamName; +} diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index e9752563f4..9da2b2be63 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -1,10 +1,3 @@ -import generateVouchers from "@cocalc/util/vouchers"; - -// nice alphanumeric string that can be used as nats subject, and very -// unlikely to randomly collide with another browser tab from this account. -export function randomId() { - return generateVouchers({ count: 1, length: 10 })[0]; -} export function handleErrorMessage(mesg) { if (mesg?.error) { @@ -15,4 +8,4 @@ export function handleErrorMessage(mesg) { } } return mesg; -} +} \ No newline at end of file diff --git a/src/packages/project/data.ts b/src/packages/project/data.ts index c96088bf39..1bf14e9a55 100644 --- a/src/packages/project/data.ts +++ b/src/packages/project/data.ts @@ -24,6 +24,7 @@ export const SSH_LOG = join(data, "sshd.log"); export const SSH_ERR = join(data, "sshd.err"); export const secretToken = process.env.COCALC_SECRET_TOKEN ?? join(data, "secret_token"); +export const compute_server_id = parseInt(process.env.COMPUTE_SERVER_ID ?? "0"); // note that the "username" need not be the output of `whoami`, e.g., // when using a cc-in-cc dev project where users are "virtual". diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index a1cd5b39df..7dbe377f1c 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -37,13 +37,13 @@ import { JSONCodec } from "nats"; import getLogger from "@cocalc/backend/logger"; import { type ProjectApi } from "@cocalc/nats/project-api"; import getConnection from "@cocalc/project/nats/connection"; -import { project_id } from "@cocalc/project/data"; +import { getSubject } from "../names"; const logger = getLogger("project:nats:api"); const jc = JSONCodec(); export async function init() { - const subject = `project.${project_id}.api`; + const subject = getSubject({ service: "api" }); logger.debug(`initAPI -- subject='${subject}'`); const nc = await getConnection(); const subscription = nc.subscribe(subject); diff --git a/src/packages/project/nats/browser-websocket-api.ts b/src/packages/project/nats/browser-websocket-api.ts index 978489a828..abc6a9bd13 100644 --- a/src/packages/project/nats/browser-websocket-api.ts +++ b/src/packages/project/nats/browser-websocket-api.ts @@ -40,11 +40,11 @@ then after that code runs you can access x from the node console! import { getLogger } from "@cocalc/project/logger"; import { JSONCodec } from "nats"; -import { project_id } from "@cocalc/project/data"; import getConnection from "./connection"; import { handleApiCall } from "@cocalc/project/browser-websocket/api"; import { getPrimusConnection } from "@cocalc/nats/primus"; import { sha1 } from "@cocalc/backend/sha1"; +import { getSubject } from "./names"; const logger = getLogger("project:nats:browser-websocket-api"); @@ -52,11 +52,15 @@ const jc = JSONCodec(); export async function init() { const nc = await getConnection(); - const subject = `project.${project_id}.browser-api`; + const subject = getSubject({ + service: "browser-api", + }); logger.debug(`initAPI -- NATS project subject '${subject}'`); const sub = nc.subscribe(subject); const primus = getPrimusConnection({ - subject: `project.${project_id}.primus`, + subject: getSubject({ + service: "primus", + }), env: { nc, sha1, jc }, role: "server", id: "project", diff --git a/src/packages/project/nats/names.ts b/src/packages/project/nats/names.ts new file mode 100644 index 0000000000..9afaf5475c --- /dev/null +++ b/src/packages/project/nats/names.ts @@ -0,0 +1,10 @@ +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { projectSubject, projectStreamName } from "@cocalc/nats/names"; + +export function getSubject(opts: { path?; service? }) { + return projectSubject({ ...opts, compute_server_id, project_id }); +} + +export function getStreamName(opts: { path?; service? }) { + return projectStreamName({ ...opts, compute_server_id, project_id }); +} diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index ac07749fe4..db52d3332a 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -9,14 +9,13 @@ import { envForSpawn } from "@cocalc/backend/misc"; import { path_split } from "@cocalc/util/misc"; import { console_init_filename, len } from "@cocalc/util/misc"; import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { project_id } from "@cocalc/project/data"; -import { sha1 } from "@cocalc/backend/sha1"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { JSONCodec } from "nats"; import { jetstreamManager } from "@nats-io/jetstream"; import { getLogger } from "@cocalc/project/logger"; import { readlink, realpath } from "node:fs/promises"; import getConnection from "./connection"; +import { getSubject, getStreamName } from "./names"; const logger = getLogger("server:nats:terminal"); @@ -108,9 +107,9 @@ class Session { MIN_KEEP, Math.min(this.options.keep ?? DEFAULT_KEEP, MAX_KEEP), ); - this.subject = `project.${project_id}.terminal.${sha1(path)}`; - this.cmd_subject = `project.${project_id}.terminal-cmd.${sha1(path)}`; - this.streamName = `project-${project_id}-terminal`; + this.subject = getSubject({ service: "terminal", path }); + this.cmd_subject = getSubject({ service: "terminal-cmd", path }); + this.streamName = getStreamName({ service: "terminal" }); } write = async (data) => { @@ -154,14 +153,14 @@ class Session { try { await jsm.streams.add({ name: this.streamName, - subjects: [`project.${project_id}.terminal.>`], + subjects: [getSubject({ service: "terminal" }) + ".>"], compression: "s2", max_msgs_per_subject: this.keep, }); } catch (_err) { // probably already exists await jsm.streams.update(this.streamName, { - subjects: [`project.${project_id}.terminal.>`], + subjects: [getSubject({ service: "terminal" }) + ".>"], compression: "s2" as any, max_msgs_per_subject: this.keep, }); From c30c16ad9881b1be156f420c1f5c1c83b6c3448c Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 31 Jan 2025 01:35:38 +0000 Subject: [PATCH 073/281] nats: converting frontend to user new project api (and also add support for compute servers) --- .../frontend/frame-editors/generic/client.ts | 2 +- .../frame-editors/latex-editor/synctex.ts | 10 +- .../frontend/project/websocket/api.ts | 208 +++++++++--------- src/packages/nats/project-api/editor.ts | 7 +- src/packages/nats/project-api/system.ts | 5 +- src/packages/project/formatters/index.ts | 3 +- src/packages/project/nats/api/editor.ts | 10 +- src/packages/project/nats/api/system.ts | 3 + src/packages/util/code-formatter.ts | 7 + src/packages/util/types/execute-code.ts | 2 +- 10 files changed, 142 insertions(+), 115 deletions(-) diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index b710191da1..c63046fb70 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -133,7 +133,7 @@ export async function formatter( const resp = await api.formatter(path, config); if (resp.status === "error") { - const loc = resp.error.loc; + const loc = resp.error?.loc; if (loc && loc.start) { throw Error( `Syntax error prevented formatting code (possibly on line ${loc.start.line} column ${loc.start.column}) -- fix and run again.`, diff --git a/src/packages/frontend/frame-editors/latex-editor/synctex.ts b/src/packages/frontend/frame-editors/latex-editor/synctex.ts index e0dcf0aedc..b73937ca6c 100644 --- a/src/packages/frontend/frame-editors/latex-editor/synctex.ts +++ b/src/packages/frontend/frame-editors/latex-editor/synctex.ts @@ -21,7 +21,7 @@ interface SyncTex { function exec_synctex( project_id: string, path: string, - args: string[] + args: string[], ): Promise { return exec({ timeout: 5, @@ -73,15 +73,13 @@ export async function tex_to_pdf(opts: { dir: string; // directory that contains the synctex file knitr: boolean; source_dir: string; + compute_server_id?: number; }): Promise { if (opts.knitr) { opts.tex_path = change_filename_extension(opts.tex_path, "Rnw"); } - // TODO: obviously this should happen once -- not constantly - // Use "available_feature.homeDirectory" instead! - const HOME = await ( - await project_api(opts.project_id) - ).eval_code("process.env.HOME"); + const projectAPI = await project_api(opts.project_id); + const HOME = await projectAPI.getHomeDirectory(opts.compute_server_id); const output = await exec_synctex(opts.project_id, opts.dir, [ "view", "-i", diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 6d3ec59ffe..4ed618e2b4 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -19,6 +19,7 @@ import { import type { Config as FormatterConfig, Options as FormatterOptions, + FormatResult, } from "@cocalc/util/code-formatter"; import { syntax2tool } from "@cocalc/util/code-formatter"; import { DirectoryListingEntry } from "@cocalc/util/types"; @@ -31,6 +32,10 @@ import type { import call from "@cocalc/sync/client/call"; import { webapp_client } from "@cocalc/frontend/webapp-client"; import { type ProjectApi } from "@cocalc/nats/project-api"; +import type { + ExecuteCodeOutput, + ExecuteCodeOptions, +} from "@cocalc/util/types/execute-code"; export class API { private conn; @@ -49,7 +54,7 @@ export class API { private getApi = ({ compute_server_id = 0, - timeout = 5000, + timeout = 15000, }: { compute_server_id?: number; timeout?: number; @@ -119,10 +124,8 @@ export class API { dest: string, compute_server_id?: number, ): Promise => { - return await this.call( - { cmd: "move_files", paths, dest, compute_server_id }, - 60000, - ); + const api = this.getApi({ compute_server_id, timeout: 60000 }); + return await api.system.moveFiles({ paths, dest }); }; // Rename the file src to be the file dest. The dest may be @@ -134,10 +137,8 @@ export class API { dest: string, compute_server_id?: number, ): Promise => { - return await this.call( - { cmd: "rename_file", src, dest, compute_server_id }, - 30000, - ); + const api = this.getApi({ compute_server_id, timeout: 60000 }); + return await api.system.renameFile({ src, dest }); }; listing = async ( @@ -146,10 +147,8 @@ export class API { timeout: number = 15000, compute_server_id: number = 0, ): Promise => { - return await this.call( - { cmd: "listing", path, hidden, compute_server_id }, - timeout, - ); + const api = this.getApi({ compute_server_id, timeout }); + return await api.system.listing({ path, hidden }); }; /* Normalize the given paths relative to the HOME directory. @@ -157,23 +156,46 @@ export class API { it one that can be opened properly with our file editor, and the path appears to be to a file *in* the HOME directory. */ - canonical_path = async (path: string): Promise => { - const v = await this.canonical_paths([path]); + canonical_path = async ( + path: string, + compute_server_id?: number, + ): Promise => { + const v = await this.canonical_paths([path], compute_server_id); const x = v[0]; if (typeof x != "string") { throw Error("bug in canonical_path"); } return x; }; - canonical_paths = async (paths: string[]): Promise => { - return await this.call({ cmd: "canonical_paths", paths }, 15000); + canonical_paths = async ( + paths: string[], + compute_server_id?: number, + ): Promise => { + const api = this.getApi({ compute_server_id }); + return await api.system.canonicalPaths(paths); }; configuration = async ( aspect: ConfigurationAspect, no_cache = false, + compute_server_id: number = 0, ): Promise => { - return await this.call({ cmd: "configuration", aspect, no_cache }, 15000); + const api = this.getApi({ compute_server_id }); + return await api.system.configuration(aspect, no_cache); + }; + + private homeDirectory: { [key: string]: string } = {}; + getHomeDirectory = async (compute_server_id: number = 0) => { + const key = `${compute_server_id}`; + if (this.homeDirectory[key] == null) { + const { capabilities } = await this.configuration( + "main", + false, + compute_server_id, + ); + this.homeDirectory[key] = capabilities.homeDirectory as string; + } + return this.homeDirectory[key]!; }; // use the returned FormatterOptions for the API formatting call! @@ -221,60 +243,97 @@ export class API { // We return a patch rather than the entire file, since often // the file is very large, but the formatting is tiny. This is purely // a data compression technique. - formatter = async (path: string, config: FormatterConfig): Promise => { + formatter = async ( + path: string, + config: FormatterConfig, + compute_server_id?: number, + ): Promise => { const options: FormatterOptions = this.check_formatter_available(config); - // TODO change this to "formatter" at some point in the future (Sep 2020) - return await this.call({ cmd: "prettier", path: path, options }, 15000); + const api = this.getApi({ compute_server_id }); + const { result } = await api.editor.formatter({ path, options }); + return result; }; formatter_string = async ( str: string, config: FormatterConfig, - timeout_ms: number = 15000, + timeout: number = 15000, + compute_server_id?: number, ): Promise => { const options: FormatterOptions = this.check_formatter_available(config); - // TODO change this to "formatter_string" at some point in the future (Sep 2020) - return await this.call( - { - cmd: "prettier_string", - str, - options, - }, - timeout_ms, - ); + const api = this.getApi({ compute_server_id, timeout }); + return await api.editor.formatterString({ str, options }); }; - jupyter = async ( - path: string, - endpoint: string, - query: any = undefined, - timeout_ms: number = 20000, - ): Promise => { - return await this.call( - { cmd: "jupyter", path, endpoint, query }, - timeout_ms, - ); - }; - - exec = async (opts: any): Promise => { + exec = async (opts: ExecuteCodeOptions): Promise => { let timeout_ms = 10000; if (opts.timeout) { + // its in seconds :-( timeout_ms = opts.timeout * 1000 + 2000; } - return await this.call({ cmd: "exec", opts }, timeout_ms); + // we explicitly remove compute_server_id since we don't need + // to pass that to opts, since exec is not proxied anymore through the project. + const { compute_server_id, ...options } = opts; + const api = this.getApi({ + compute_server_id, + timeout: timeout_ms, + }); + return await api.system.exec(options); + }; + + realpath = async ( + path: string, + compute_server_id?: number, + ): Promise => { + const api = this.getApi({ compute_server_id }); + return await api.system.realpath(path); }; - eval_code = async ( - code: string, - timeout_ms: number = 20000, + // Convert a notebook to some other format. + // --to options are listed in packages/frontend/jupyter/nbconvert.tsx + // and implemented in packages/project/jupyter/convert/index.ts + jupyter_nbconvert = async ( + opts: NbconvertParams, + compute_server_id?: number, ): Promise => { - return await this.call({ cmd: "eval_code", code }, timeout_ms); + const api = this.getApi({ + compute_server_id, + timeout: (opts.timeout ?? 60) * 1000 + 5000, + }); + return await api.editor.jupyterNbconvert(opts); }; - realpath = async (path: string): Promise => { - return await this.call({ cmd: "realpath", path }, 15000); + // Get contents of an ipynb file, but with output and attachments removed (to save space) + jupyter_strip_notebook = async ( + ipynb_path: string, + compute_server_id?: number, + ): Promise => { + const api = this.getApi({ compute_server_id }); + return await api.editor.jupyterStripNotebook(ipynb_path); }; + // Run the notebook filling in the output of all cells, then return the + // result as a string. Note that the output size (per cell and total) + // and run time is bounded to avoid the output being HUGE, even if the + // input is dumb. + + jupyter_run_notebook = async ( + opts: RunNotebookOptions, + compute_server_id?: number, + ): Promise => { + const max_total_time_ms = opts.limits?.max_total_time_ms ?? 20 * 60 * 1000; + // a bit of extra time -- it's better to let the internal project + // timer do the job, than have to wait for this generic timeout here, + // since we want to at least get output for problems that ran. + const api = this.getApi({ + compute_server_id, + timeout: 60 + 2 * max_total_time_ms, + }); + return await api.editor.jupyterRunNotebook(opts); + }; + + // TODO! + terminal = async (path: string, options: object = {}): Promise => { const channel_name = await this.call( { @@ -353,40 +412,6 @@ export class API { return await this.call({ cmd: "lean", opts }, timeout_ms); }; - // Convert a notebook to some other format. - // --to options are listed in packages/frontend/jupyter/nbconvert.tsx - // and implemented in packages/project/jupyter/convert/index.ts - jupyter_nbconvert = async (opts: NbconvertParams): Promise => { - return await this.call( - { cmd: "jupyter_nbconvert", opts }, - (opts.timeout ?? 60) * 1000 + 5000, - ); - }; - - // Get contents of an ipynb file, but with output and attachments removed (to save space) - jupyter_strip_notebook = async (ipynb_path: string): Promise => { - return await this.call( - { cmd: "jupyter_strip_notebook", ipynb_path }, - 15000, - ); - }; - - // Run the notebook filling in the output of all cells, then return the - // result as a string. Note that the output size (per cell and total) - // and run time is bounded to avoid the output being HUGE, even if the - // input is dumb. - - jupyter_run_notebook = async (opts: RunNotebookOptions): Promise => { - const max_total_time_ms = opts.limits?.max_total_time_ms ?? 20 * 60 * 1000; - return await this.call( - { cmd: "jupyter_run_notebook", opts }, - 60 + 2 * max_total_time_ms, - // a bit of extra time -- it's better to let the internal project - // timer do the job, than have to wait for this generic timeout here, - // since we want to at least get output for problems that ran. - ); - }; - // I think this isn't used. It was going to support // sync_channel, but obviously a more nuanced protocol // was required. @@ -401,19 +426,6 @@ export class API { return this.conn.channel(channel_name); }; - // Do a database query, but via the project. This has the project - // do the query, so the identity used to access the database is that - // of the project. This isn't useful in the browser, where the user - // always has more power to directly use the database. It is *is* - // very useful when using a project-specific api key. - query = async (opts: any): Promise => { - if (opts.timeout == null) { - opts.timeout = 30; - } - const timeout_ms = opts.timeout * 1000 + 2000; - return await this.call({ cmd: "query", opts }, timeout_ms); - }; - computeServerSyncRequest = async (compute_server_id: number) => { if (!(typeof compute_server_id == "number" && compute_server_id > 0)) { throw Error("compute_server_id must be a positive integer"); diff --git a/src/packages/nats/project-api/editor.ts b/src/packages/nats/project-api/editor.ts index 8571fcf599..2389c4ba57 100644 --- a/src/packages/nats/project-api/editor.ts +++ b/src/packages/nats/project-api/editor.ts @@ -1,6 +1,9 @@ import type { NbconvertParams } from "@cocalc/util/jupyter/types"; import type { RunNotebookOptions } from "@cocalc/util/jupyter/nbgrader-types"; -import type { Options as FormatterOptions } from "@cocalc/util/code-formatter"; +import type { + Options as FormatterOptions, + FormatResult, +} from "@cocalc/util/code-formatter"; export const editor = { jupyterStripNotebook: true, @@ -20,7 +23,7 @@ export interface Editor { formatter: (opts: { path: string; options: FormatterOptions; - }) => Promise; + }) => Promise<{ result: FormatResult }>; formatterString: (opts: { str: string; diff --git a/src/packages/nats/project-api/system.ts b/src/packages/nats/project-api/system.ts index 11b727605f..a7b6d9f685 100644 --- a/src/packages/nats/project-api/system.ts +++ b/src/packages/nats/project-api/system.ts @@ -17,13 +17,13 @@ export const system = { deleteFiles: true, moveFiles: true, renameFile: true, + realpath: true, canonicalPaths: true, configuration: true, ping: true, exec: true, - realpath: true, }; export interface System { @@ -38,6 +38,8 @@ export interface System { deleteFiles: (opts: { paths: string[] }) => Promise; moveFiles: (opts: { paths: string[]; dest: string }) => Promise; renameFile: (opts: { src: string; dest: string }) => Promise; + realpath: (path: string) => Promise; + canonicalPaths: (paths: string[]) => Promise; configuration: ( aspect: ConfigurationAspect, @@ -48,5 +50,4 @@ export interface System { exec: (opts: ExecuteCodeOptions) => Promise; - realpath: (path: string) => Promise; } diff --git a/src/packages/project/formatters/index.ts b/src/packages/project/formatters/index.ts index d255325e76..c9531e7950 100644 --- a/src/packages/project/formatters/index.ts +++ b/src/packages/project/formatters/index.ts @@ -35,6 +35,7 @@ import type { Syntax as FormatterSyntax, Config, Options, + FormatResult, } from "@cocalc/util/code-formatter"; export type { Config, Options, FormatterSyntax }; import { getLogger } from "@cocalc/backend/logger"; @@ -48,7 +49,7 @@ export async function run_formatter({ }: { path: string; options: Options; -}): Promise { +}): Promise { const client = getClient(); // What we do is edit the syncstring with the given path to be "prettier" if possible... const syncstring = client.syncdoc({ path }); diff --git a/src/packages/project/nats/api/editor.ts b/src/packages/project/nats/api/editor.ts index bda781480f..c79bf3939e 100644 --- a/src/packages/project/nats/api/editor.ts +++ b/src/packages/project/nats/api/editor.ts @@ -1,7 +1,9 @@ export { jupyter_strip_notebook as jupyterStripNotebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; export { jupyter_run_notebook as jupyterRunNotebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; export { nbconvert as jupyterNbconvert } from "../../jupyter/convert"; -export { - run_formatter as formatter, - run_formatter_string as formatterString, -} from "../../formatters"; + +export { run_formatter_string as formatterString } from "../../formatters"; +import { run_formatter } from "../../formatters"; +export async function formatter(opts) { + return { result: await run_formatter(opts) }; +} diff --git a/src/packages/project/nats/api/system.ts b/src/packages/project/nats/api/system.ts index 9c1e29d04b..0321e2b2cf 100644 --- a/src/packages/project/nats/api/system.ts +++ b/src/packages/project/nats/api/system.ts @@ -49,3 +49,6 @@ export async function renameFile({ src, dest }: { src: string; dest: string }) { import { get_configuration } from "@cocalc/project/configuration"; export { get_configuration as configuration }; + +import { canonical_paths } from "../../browser-websocket/canonical-path"; +export { canonical_paths as canonicalPaths }; diff --git a/src/packages/util/code-formatter.ts b/src/packages/util/code-formatter.ts index a5946c35d2..3274f780d3 100644 --- a/src/packages/util/code-formatter.ts +++ b/src/packages/util/code-formatter.ts @@ -251,3 +251,10 @@ export interface Options extends Omit { parser: Syntax; // TODO refactor this to tool tabWidth?: number; } + +export interface FormatResult { + status: "ok" | "error"; + patch?: any; + phase?: string; + error?: any; +} diff --git a/src/packages/util/types/execute-code.ts b/src/packages/util/types/execute-code.ts index 3b72130e1b..452bdccd78 100644 --- a/src/packages/util/types/execute-code.ts +++ b/src/packages/util/types/execute-code.ts @@ -35,7 +35,7 @@ export interface ExecuteCodeOptions { command: string; args?: string[]; path?: string; // defaults to home directory; where code is executed from. absolute path or path relative to home directory. - timeout?: number; // timeout in *seconds* + timeout?: number; // timeout in **seconds** ulimit_timeout?: boolean; // If set (the default), use ulimit to ensure a cpu timeout -- don't use when launching a daemon! // This has no effect if bash not true. err_on_exit?: boolean; // if true (the default), then a nonzero exit code will result in an error; if false, even with a nonzero exit code you just get back the stdout, stderr and the exit code as usual. From 1c4dbdce8e1a0246c68dbf5c5f4f1d5892816c26 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 31 Jan 2025 04:50:25 +0000 Subject: [PATCH 074/281] nats: implement for kv for tracking open-files --- .../frontend/project/websocket/api.ts | 1 - src/packages/nats/sync/open-files.ts | 113 ++++++++++++++++++ src/packages/sync/editor/generic/sync-doc.ts | 4 +- 3 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 src/packages/nats/sync/open-files.ts diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 4ed618e2b4..ef320dca10 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -333,7 +333,6 @@ export class API { }; // TODO! - terminal = async (path: string, options: object = {}): Promise => { const channel_name = await this.call( { diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts new file mode 100644 index 0000000000..dae69c500e --- /dev/null +++ b/src/packages/nats/sync/open-files.ts @@ -0,0 +1,113 @@ +/* +NATS Kv associated to a project to keep track of open files. + +DEVELOPMENT: + +~/cocalc/src/packages/project$ node +> z = new (require('@cocalc/nats/sync/open-files').OpenFiles)({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf', env:await require('@cocalc/backend/nats').getEnv()}) +> await z.set({path:'a.txt',interest:Date.now(),open:true,id:1}) +> await z.get() +{ + 'a.txt': { path: 'a.txt', interest: 1738298844728, open: true, id: 1 } +} +> await z.set({path:'foo/b.md',interest:Date.now(),open:true,id:0}) +undefined +> await z.get() +{ + 'a.txt': { path: 'a.txt', interest: 1738298844728, open: true, id: 1 }, + 'foo/b.md': { path: 'foo/b.md', interest: 1738298896539, open: true, id: 0 } +} +> await z.get('foo/b.dm') +null +> await z.get('foo/b.md') +{ path: 'foo/b.md', interest: 1738298896539, open: true, id: 0 } +*/ + +import { getKv, type NatsEnv } from "./synctable-kv"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { sha1 } from "@cocalc/util/misc"; + +const PREFIX = `open-files`; + +interface Entry { + // path to file relative to HOME + path: string; + // compute server id or 0/not defined for home base + id?: number; + // if true, then file should be opened, managed, and watched + // by home base or compute server + open?: boolean; + // last time a client expressed interest in the file + interest?: number; +} + +export class OpenFiles { + private kv?; + private nc; + private jc; + private sha1; + private project_id: string; + + constructor({ env, project_id }: { env: NatsEnv; project_id: string }) { + this.sha1 = env.sha1 ?? sha1; + this.nc = env.nc; + this.jc = env.jc; + this.project_id = project_id; + } + + private getKv = reuseInFlight(async () => { + if (this.kv == null) { + this.kv = await getKv({ + nc: this.nc, + project_id: this.project_id, + }); + } + return this.kv!; + }); + + private getKey = ({ path }: Entry): string => { + return `${PREFIX}.${this.sha1(path)}`; + }; + + set = async (obj: Entry) => { + const key = this.getKey(obj); + const value = this.jc.encode(obj); + const kv = await this.getKv(); + await kv.put(key, value); + }; + + delete = async (obj) => { + const kv = await this.getKv(); + await kv.delete(this.getKey(obj)); + }; + + private decode = (mesg) => { + return mesg?.sm?.data != null ? this.jc.decode(mesg.sm.data) : null; + }; + + get = async (path?: string) => { + const kv = await this.getKv(); + if (path == null) { + // everything + const keys = await kv.keys(`${PREFIX}.>`); + const all: { [path: string]: Entry } = {}; + for await (const key of keys) { + const obj = this.decode(await kv.get(key)); + all[obj.path] = obj; + } + return all; + } + return this.decode(await kv.get(this.getKey({ path }))); + }; + + // watch for changes + async *watch() { + const kv = await this.getKv(); + const w = await kv.watch({ + key: `${PREFIX}.>`, + }); + for await (const { value } of w) { + yield this.jc.decode(value); + } + } +} diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index f6499464d0..6ed7bda3d8 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -19,7 +19,7 @@ EVENTS: - ... TODO */ -const USE_NATS = false; +const USE_NATS = true; /* OFFLINE_THRESH_S - If the client becomes disconnected from the backend for more than this long then---on reconnect---do @@ -277,7 +277,7 @@ export class SyncDoc extends EventEmitter { this[field] = opts[field]; } } - this.useNats = USE_NATS && this.path.startsWith("nats/"); + this.useNats = USE_NATS; if (this.ephemeral) { // So the doctype written to the database reflects the // ephemeral state. Here ephemeral determines whether From 2b20b905e5cb0b5f16ef70737b2c606454b2a60f Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 31 Jan 2025 14:52:47 +0000 Subject: [PATCH 075/281] nats OpenFiles tracker -- more work on it --- src/packages/frontend/client/nats.ts | 12 ++ src/packages/nats/sync/open-files.ts | 181 +++++++++++++++++++------ src/packages/nats/sync/synctable-kv.ts | 4 +- src/packages/server/nats/auth.ts | 9 +- 4 files changed, 163 insertions(+), 43 deletions(-) diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index e6bf5b58c9..01da3efc72 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -14,6 +14,7 @@ import { type HubApi, initHubApi } from "@cocalc/nats/hub-api"; import { type ProjectApi, initProjectApi } from "@cocalc/nats/project-api"; import { getPrimusConnection } from "@cocalc/nats/primus"; import { isValidUUID } from "@cocalc/util/misc"; +import { OpenFiles } from "@cocalc/nats/sync/open-files"; export class NatsClient { /*private*/ client: WebappClient; @@ -24,6 +25,7 @@ export class NatsClient { public jetstream = jetstream; public hub: HubApi; public sessionId = randomId(); + private openFilesCache: { [project_id: string]: OpenFiles } = {}; constructor(client: WebappClient) { this.client = client; @@ -230,4 +232,14 @@ export class NatsClient { id: this.sessionId, }); }; + + openFiles = reuseInFlight(async (project_id: string) => { + if (this.openFilesCache[project_id] == null) { + this.openFilesCache[project_id] = new OpenFiles({ + project_id, + env: await this.getEnv(), + }); + } + return this.openFilesCache[project_id]!; + }); } diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index dae69c500e..95d6bd89b5 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -3,29 +3,35 @@ NATS Kv associated to a project to keep track of open files. DEVELOPMENT: +Change to packages/project, since packages/nats doesn't have a way to connect: + ~/cocalc/src/packages/project$ node > z = new (require('@cocalc/nats/sync/open-files').OpenFiles)({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf', env:await require('@cocalc/backend/nats').getEnv()}) -> await z.set({path:'a.txt',interest:Date.now(),open:true,id:1}) +> await z.touch({path:'a.txt',id:1}) > await z.get() { - 'a.txt': { path: 'a.txt', interest: 1738298844728, open: true, id: 1 } + 'a.txt': { path: 'a.txt', open: true, id: 1, time: 2025-01-31T14:16:48.314Z } } -> await z.set({path:'foo/b.md',interest:Date.now(),open:true,id:0}) +> await z.touch({path:'foo/b.md',id:0}) undefined > await z.get() { - 'a.txt': { path: 'a.txt', interest: 1738298844728, open: true, id: 1 }, - 'foo/b.md': { path: 'foo/b.md', interest: 1738298896539, open: true, id: 0 } + 'a.txt': { path: 'a.txt', interest: 1738298844728, open: true, id: 1, time: 2025-01-31T14:16:48.314Z }, + 'foo/b.md': { path: 'foo/b.md', interest: 1738298896539, open: true, id: 0, time:... } } -> await z.get('foo/b.dm') +> await z.get({path:'foo/b.dm'}) null -> await z.get('foo/b.md') -{ path: 'foo/b.md', interest: 1738298896539, open: true, id: 0 } +> await z.get({path:'foo/b.md'}) +{ path: 'foo/b.md', open: true, id: 0 } + +> for await (const x of await z.watch()) { console.log(x)} + */ import { getKv, type NatsEnv } from "./synctable-kv"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { sha1 } from "@cocalc/util/misc"; +import { isEqual } from "lodash"; const PREFIX = `open-files`; @@ -37,8 +43,33 @@ interface Entry { // if true, then file should be opened, managed, and watched // by home base or compute server open?: boolean; - // last time a client expressed interest in the file - interest?: number; + // last time this entry was changed -- this is automatically set + // correctly by the NATS server in a consistent way: + // https://github.com/nats-io/nats-server/discussions/3095 + // It gets updated even if you set an object to itself (making no change). + time?: Date; +} + +const FIELDS = ["path", "id", "open"]; + +function validObject(obj: Entry) { + const obj2: any = {}; + for (const field of FIELDS) { + const val = obj[field]; + if (val != null) { + if (field == "path") { + obj2[field] = typeof val == "string" ? val : `${val}`; + } else if (field == "id") { + obj2[field] = typeof val == "number" ? val : parseInt(`${val}`); + } else if (field == "open") { + obj2[field] = typeof val == "boolean" ? val : !!val; + } + } + } + if (!obj2["path"]) { + throw Error("path must be specified"); + } + return obj2; } export class OpenFiles { @@ -55,39 +86,47 @@ export class OpenFiles { this.project_id = project_id; } - private getKv = reuseInFlight(async () => { - if (this.kv == null) { - this.kv = await getKv({ - nc: this.nc, - project_id: this.project_id, - }); - } - return this.kv!; - }); - - private getKey = ({ path }: Entry): string => { - return `${PREFIX}.${this.sha1(path)}`; - }; - - set = async (obj: Entry) => { + // When a client has a file open, they should periodically + // touch it to indicate that it is open. + // updates timestamp and ensures open=true. + touch = async (obj0: { path: string; id?: number }) => { + // just read and write it back, which updates the timestamp + // no encode/decode needed. + const obj = { ...validObject(obj0), open: true }; const key = this.getKey(obj); - const value = this.jc.encode(obj); const kv = await this.getKv(); - await kv.put(key, value); + const mesg = await kv.get(key); + if (mesg == null || mesg.sm.data.length == 0) { + // no current entry -- create new + await this.set(obj); + } else { + const cur = this.decode(mesg, true); + const newValue = { ...cur, ...obj }; + if (!isEqual(cur, newValue)) { + await this.set(newValue); + } else { + // update existing by just rewriting it back; this updates timestamp too + await kv.put(key, mesg.sm.data); + } + } }; - delete = async (obj) => { + close = async ({ path }: { path: string }) => { const kv = await this.getKv(); - await kv.delete(this.getKey(obj)); - }; - - private decode = (mesg) => { - return mesg?.sm?.data != null ? this.jc.decode(mesg.sm.data) : null; + const key = this.getKey({ path }); + const mesg = await kv.get(key); + if (mesg.sm.data.length == 0) { + // nothing to do + return; + } + const cur = this.decode(mesg, true); + const value = this.jc.encode({ ...cur, open: false }); + await kv.put(key, value); }; - get = async (path?: string) => { + get = async (obj?: Entry) => { const kv = await this.getKv(); - if (path == null) { + if (obj == null) { // everything const keys = await kv.keys(`${PREFIX}.>`); const all: { [path: string]: Entry } = {}; @@ -97,7 +136,7 @@ export class OpenFiles { } return all; } - return this.decode(await kv.get(this.getKey({ path }))); + return this.decode(await kv.get(this.getKey(validObject(obj)))); }; // watch for changes @@ -105,9 +144,75 @@ export class OpenFiles { const kv = await this.getKv(); const w = await kv.watch({ key: `${PREFIX}.>`, + // we assume that we ONLY delete old items which are not relevant + ignoreDeletes: true, }); - for await (const { value } of w) { - yield this.jc.decode(value); + for await (const mesg of w) { + // no need to check for 'mesg.value.length' due to ignoreDeletes above. + yield this.decode(mesg); } } + + // delete entries that haven't been touched in ageMs milliseconds. + // default=a month + // returns number of deleted objects. + deleteOld = async (ageMs: number = 1000 * 60 * 60 * 730): Promise => { + let n = 0; + const cutoff = new Date(Date.now() - ageMs); + const kv = await this.getKv(); + const keys = await kv.keys(`${PREFIX}.>`); + for await (const key of keys) { + const mesg = await kv.get(key); + if (mesg.sm.time <= cutoff) { + await kv.delete(key); + n += 1; + } + } + return n; + }; + + // dangerous - e.g., our watcher assumes no deletes. Instead, you should + // close files, not delete. + delete = async (obj0) => { + const obj = validObject(obj0); + const kv = await this.getKv(); + await kv.delete(this.getKey(obj)); + }; + + has = async ({ path }): Promise => { + const kv = await this.getKv(); + const key = this.getKey({ path }); + const mesg = await kv.get(key); + return mesg.sm.data.length > 0; + }; + + private getKv = reuseInFlight(async () => { + if (this.kv == null) { + this.kv = await getKv({ + nc: this.nc, + project_id: this.project_id, + }); + } + return this.kv!; + }); + + private getKey = ({ path }: Entry): string => { + return `${PREFIX}.${this.sha1(path)}`; + }; + + // atomic set - NOT a merge set. + private set = async (obj0: Entry) => { + let obj = validObject(obj0); + const key = this.getKey(obj); + const value = this.jc.encode(obj); + const kv = await this.getKv(); + await kv.put(key, value); + }; + + private decode = (mesg, noDate = false): Entry => { + return { + ...this.jc.decode(mesg.sm.data), + ...(noDate ? undefined : { time: mesg.sm.time }), + }; + }; } diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 30492775e2..ef8dc7d41a 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -28,10 +28,12 @@ export async function getKv({ nc, project_id, account_id, + options, }: { nc; project_id?: string; account_id?: string; + options?; }) { let name; if (account_id) { @@ -42,7 +44,7 @@ export async function getKv({ throw Error("one of account_id or project_id must be defined"); } const kvm = new Kvm(nc); - return await kvm.create(name, { compression: true }); + return await kvm.create(name, { compression: true, ...options }); } export interface NatsEnv { diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 1c91ce9fdd..7afb18705a 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -18,7 +18,8 @@ DOCS: USAGE: -a = require('@cocalc/server/nats/auth'); await a.configureNatsUser({account_id:'275f1db7-bf37-4b44-b9aa-d64694269c9f'}) +a = require('@cocalc/server/nats/auth'); +await a.configureNatsUser({account_id:'275f1db7-bf37-4b44-b9aa-d64694269c9f'}) await a.configureNatsUser({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}) */ @@ -108,7 +109,7 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { throw Error("must be a valid uuid"); } const userType = getCoCalcUserType(cocalcUser); - // TODO: jetstream permissions are WAY TO BROAD. + // TODO: jetstream permissions are WAY TO BROAD. const goalPub = new Set([ "_INBOX.>", `hub.${userType}.${userId}.>`, @@ -225,9 +226,9 @@ export async function addProjectPermission({ account_id, project_id }) { "--sk", name, "--allow-sub", - `project.${project_id}.>,*.project.${project_id}.>`, + `project.${project_id}.>,*.project-${project_id}.>`, "--allow-pub", - `project.${project_id}.>,*.project.${project_id}.>`, + `project.${project_id}.>,*.project-${project_id}.>`, ]); await pushToServer(); } From 72c4d2e7007468c3a7e1d3a4393fde3b19d65c43 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 31 Jan 2025 15:58:55 +0000 Subject: [PATCH 076/281] nats sync: when client browser has document open, it periodically updates the open_files kv store --- src/packages/frontend/client/client.ts | 13 ++++++++++++ src/packages/sync/editor/generic/sync-doc.ts | 22 ++++++++++++++++++-- src/packages/sync/editor/generic/types.ts | 2 ++ src/packages/util/nats.ts | 4 ++++ 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 src/packages/util/nats.ts diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 3ba556a8fe..84d9dbdd64 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -349,6 +349,19 @@ class Client extends EventEmitter implements WebappClient { public set_deleted(): void { throw Error("not implemented for frontend"); } + + touchOpenFile = async ({ + project_id, + path, + id, + }: { + project_id: string; + path: string; + id?: number; + }) => { + const x = await this.nats_client.openFiles(project_id); + await x.touch({ path, id }); + }; } export const webapp_client = new Client(); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 6ed7bda3d8..b7531b3305 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -109,6 +109,7 @@ import { Patch, } from "./types"; import { patch_cmp } from "./util"; +import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; export type State = "init" | "ready" | "closed"; export type DataServer = "project" | "database"; @@ -324,7 +325,7 @@ export class SyncDoc extends EventEmitter { until it is (however long, etc.). If this fails, it closes this SyncDoc. */ - private async init(): Promise { + private init = async (): Promise => { this.assert_not_closed("init"); const log = this.dbg("init"); @@ -356,7 +357,7 @@ export class SyncDoc extends EventEmitter { this.set_state("ready"); this.init_watch(); this.emit_change(); // from nothing to something. - } + }; // True if this client is responsible for managing // the state of this document with respect to @@ -1357,6 +1358,9 @@ export class SyncDoc extends EventEmitter { } const log = this.dbg("init_all"); + log("update interest"); + this.initInterestLoop(); + log("ensure syncstring exists in database"); this.assert_not_closed("init_all -- before ensuring syncstring exists"); await this.ensure_syncstring_exists_in_db(); @@ -3378,4 +3382,18 @@ export class SyncDoc extends EventEmitter { await this.syncstring_table.save(); } }; + + private initInterestLoop = async () => { + if (!this.client.is_browser() || this.client.touchOpenFile == null) { + // only browser clients -- so actual humans + return; + } + while (this.state != "closed") { + await this.client.touchOpenFile({ + path: this.path, + project_id: this.project_id, + }); + await delay(NATS_OPEN_FILE_TOUCH_INTERVAL); + } + }; } diff --git a/src/packages/sync/editor/generic/types.ts b/src/packages/sync/editor/generic/types.ts index e083b22d42..ae04018ced 100644 --- a/src/packages/sync/editor/generic/types.ts +++ b/src/packages/sync/editor/generic/types.ts @@ -138,6 +138,8 @@ export interface Client extends ProjectClient { shell: (opts: ExecuteCodeOptionsWithCallback) => void; sage_session: (opts: { path: string }) => any; + + touchOpenFile?: (opts: { project_id: string; path: string }) => Promise; } export interface DocType { diff --git a/src/packages/util/nats.ts b/src/packages/util/nats.ts new file mode 100644 index 0000000000..e80be8fc26 --- /dev/null +++ b/src/packages/util/nats.ts @@ -0,0 +1,4 @@ +// Some very generic nats related parameters + +// how frequently +export const NATS_OPEN_FILE_TOUCH_INTERVAL = 30000; From 47f026c5d44df1e9151ae7d5345cf037d7dbb9c6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 31 Jan 2025 17:18:27 +0000 Subject: [PATCH 077/281] nats open files tracker -- working on it --- src/packages/nats/sync/open-files.ts | 18 ++- src/packages/project/nats/env.ts | 9 ++ src/packages/project/nats/open-files.ts | 133 +++++++++++++++++++ src/packages/sync/editor/generic/sync-doc.ts | 11 +- src/packages/sync/editor/generic/types.ts | 6 +- 5 files changed, 168 insertions(+), 9 deletions(-) create mode 100644 src/packages/project/nats/env.ts create mode 100644 src/packages/project/nats/open-files.ts diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 95d6bd89b5..c2c3b59058 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -35,7 +35,7 @@ import { isEqual } from "lodash"; const PREFIX = `open-files`; -interface Entry { +export interface Entry { // path to file relative to HOME path: string; // compute server id or 0/not defined for home base @@ -78,6 +78,8 @@ export class OpenFiles { private jc; private sha1; private project_id: string; + public state: "ready" | "closed" = "ready"; + private watches: any[] = []; constructor({ env, project_id }: { env: NatsEnv; project_id: string }) { this.sha1 = env.sha1 ?? sha1; @@ -86,6 +88,14 @@ export class OpenFiles { this.project_id = project_id; } + close = () => { + this.state = "closed"; + for (const w of this.watches) { + w.close(); + this.watches = []; + } + }; + // When a client has a file open, they should periodically // touch it to indicate that it is open. // updates timestamp and ensures open=true. @@ -111,7 +121,7 @@ export class OpenFiles { } }; - close = async ({ path }: { path: string }) => { + closeFile = async ({ path }: { path: string }) => { const kv = await this.getKv(); const key = this.getKey({ path }); const mesg = await kv.get(key); @@ -147,9 +157,13 @@ export class OpenFiles { // we assume that we ONLY delete old items which are not relevant ignoreDeletes: true, }); + this.watches.push(w); for await (const mesg of w) { // no need to check for 'mesg.value.length' due to ignoreDeletes above. yield this.decode(mesg); + if (this.state == "closed") { + return; + } } } diff --git a/src/packages/project/nats/env.ts b/src/packages/project/nats/env.ts new file mode 100644 index 0000000000..8aa9f5dfec --- /dev/null +++ b/src/packages/project/nats/env.ts @@ -0,0 +1,9 @@ +import { sha1 } from "@cocalc/backend/sha1"; +import getConnection from "./connection"; +import { JSONCodec } from "nats"; + +export async function getEnv() { + const nc = await getConnection(); + const jc = JSONCodec(); + return { sha1, nc, jc }; +} diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts new file mode 100644 index 0000000000..5edb602cfb --- /dev/null +++ b/src/packages/project/nats/open-files.ts @@ -0,0 +1,133 @@ +/* +Handle opening files in a project to save/load from disk and also enable compute capabilities. + +DEVELOPMENT: + +Set env variables as in a project, then: + +> require("@cocalc/project/nats/open-files").init() +*/ + +import { OpenFiles, Entry } from "@cocalc/nats/sync/open-files"; +import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { getEnv } from "./env"; +import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; +import { getClient } from "@cocalc/project/client"; +//import { SyncDB } from "@cocalc/sync/editor/db/sync"; +import { SyncString } from "@cocalc/sync/editor/string/sync"; +import getLogger from "@cocalc/backend/logger"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { delay } from "awaiting"; + +const logger = getLogger("project:nats:open-files"); + +export async function init() { + logger.debug("init"); + const openFiles = new OpenFiles({ + project_id, + env: await getEnv(), + }); + const entries: { [path: string]: Entry } = {}; + closeIgnoredFiles(entries, openFiles); + for await (const entry of await openFiles.watch()) { + entries[entry.path] = entry; + await handleEntry(entry); + } +} + +const openSyncDocs: { [path: string]: SyncDoc } = {}; +// for dev +export { openSyncDocs }; + +function getCutoff() { + return new Date(Date.now() - 2.5 * NATS_OPEN_FILE_TOUCH_INTERVAL); +} + +async function handleEntry({ path, id = 0, open, time }: Entry) { + const syncDoc = openSyncDocs[path]; + const isOpenHere = syncDoc != null; + if (id != compute_server_id) { + if (isOpenHere) { + // close it here + closeSyncDoc(path); + } + // no further responsibility + return; + } + if (!open) { + if (isOpenHere) { + closeSyncDoc(path); + } + return; + } + if (time != null && open && time >= getCutoff()) { + if (!isOpenHere) { + // users actively care about this file being opened HERE, but it isn't + openSyncDoc(path); + } + return; + } +} + +function supportAutoclose(path: string) { + // this feels way too "hard coded"; alternatively, maybe we make the kernel or whatever + // actually update the interest? or something else... + if (path.endsWith(".ipynb.sage-jupyter2") || path.endsWith(".sagews")) { + return false; + } + return true; +} + +async function closeIgnoredFiles(entries, openFiles) { + while (openFiles.state == "ready") { + await delay(NATS_OPEN_FILE_TOUCH_INTERVAL); + if (openFiles.state != "ready") { + return; + } + logger.debug("closeIgnoredFiles: checking..."); + const cutoff = getCutoff(); + for (const path in entries) { + const entry = entries[path]; + if ( + entry.time <= cutoff && + !supportAutoclose(path) && + openSyncDocs[path] != null + ) { + logger.debug("closeIgnoredFiles: closing due to inactivity", { path }); + closeSyncDoc(path); + } + } + } +} + +const closeSyncDoc = reuseInFlight(async (path: string) => { + logger.debug("close", { path }); + const syncDoc = openSyncDocs[path]; + if (syncDoc == null) { + return; + } + delete openSyncDocs[path]; + try { + await syncDoc.close(); + } catch (err) { + // TODO: maybe this could get saved in a nats key-value store? + logger.debug(`WARNING -- issue closing syncdoc -- ${err}`); + } +}); + +const openSyncDoc = reuseInFlight(async (path: string) => { + // todo -- will be async and needs to handle SyncDB and all the config... + logger.debug("open", { path }); + const syncDoc = openSyncDocs[path]; + if (syncDoc != null) { + return syncDoc; + } + const client = getClient(); + openSyncDocs[path] = new SyncString({ + project_id, + path, + client, + }); + return openSyncDocs[path]!; +}); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index b7531b3305..06fd0e9c24 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -635,7 +635,7 @@ export class SyncDoc extends EventEmitter { return; } this.last_user_change = new Date(); - this.client.mark_file({ + this.client.mark_file?.({ project_id: this.project_id, path: this.path, action: "edit", @@ -1272,7 +1272,10 @@ export class SyncDoc extends EventEmitter { ); break; case "database": - synctable = await this.client.synctable_database( + if (this.client.synctable_database == null) { + throw Error("database server not supported by project"); + } + synctable = await this.client.synctable_database?.( query, options, throttle_changes, @@ -3065,8 +3068,8 @@ export class SyncDoc extends EventEmitter { ) { return; } - if (last_err && typeof this.client.log_error === "function") { - this.client.log_error({ + if (last_err && typeof this.client.log_error != null) { + this.client.log_error?.({ string_id: this.string_id, path: this.path, project_id: this.project_id, diff --git a/src/packages/sync/editor/generic/types.ts b/src/packages/sync/editor/generic/types.ts index ae04018ced..942235ac88 100644 --- a/src/packages/sync/editor/generic/types.ts +++ b/src/packages/sync/editor/generic/types.ts @@ -115,21 +115,21 @@ export interface ProjectClient extends EventEmitter { } export interface Client extends ProjectClient { - log_error: (opts: { + log_error?: (opts: { project_id: string; path: string; string_id: string; error: any; }) => void; - mark_file: (opts: { + mark_file?: (opts: { project_id: string; path: string; action: string; ttl: number; }) => void; - synctable_database: ( + synctable_database?: ( query: any, options: any, throttle_changes?: number, From 5f696a7078ca2db731538e385451227f9315f19d Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 31 Jan 2025 18:38:14 +0000 Subject: [PATCH 078/281] nats synctable-kv -- work in progress --- src/packages/nats/sync/synctable-kv.ts | 139 ++++++++++++++++++--- src/packages/nats/sync/synctable-stream.ts | 8 +- 2 files changed, 125 insertions(+), 22 deletions(-) diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index ef8dc7d41a..8f10c7ffc5 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -13,6 +13,9 @@ import { sha1 } from "@cocalc/util/misc"; import jsonStableStringify from "json-stable-stringify"; import { keys } from "lodash"; import { client_db } from "@cocalc/util/db-schema/client-db"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { EventEmitter } from "events"; +import { wait } from "@cocalc/util/async-wait"; export function natsKeyPrefix({ query, @@ -36,10 +39,10 @@ export async function getKv({ options?; }) { let name; - if (account_id) { - name = `account-${account_id}`; - } else if (project_id) { + if (project_id) { name = `project-${project_id}`; + } else if (account_id) { + name = `account-${account_id}`; } else { throw Error("one of account_id or project_id must be defined"); } @@ -64,7 +67,7 @@ export function toKey(x): string | undefined { } } -export class SyncTableKV { +export class SyncTableKV extends EventEmitter { private kv?; private nc; private jc; @@ -76,6 +79,9 @@ export class SyncTableKV { private fields: string[]; private project_id?: string; private account_id?: string; + private data: { [key: string]: any } = {}; + private state: "disconnected" | "connected" | "closed" = "disconnected"; + private updateListener?; constructor({ query, @@ -88,6 +94,7 @@ export class SyncTableKV { account_id?: string; project_id?: string; }) { + super(); this.sha1 = env.sha1 ?? sha1; this.nc = env.nc; this.jc = env.jc; @@ -101,14 +108,104 @@ export class SyncTableKV { this.fields = keys(query[table][0]).filter( (field) => !this.primaryKeysSet.has(field), ); + this.readData(); + } + + get = (obj?) => { + if (this.state != "connected") { + throw Error("must be connected"); + } + if (obj == null) { + const result: any = {}; + for (const k in this.data) { + result[this.primaryString(this.data[k])] = this.data[k]; + } + return result; + } + return this.data[this.getKey(obj)]; + }; + + get_one = () => { + for (const key in this.data) { + return this.data[key]; + } + }; + + set = (obj) => { + const key = this.getKey(obj); + this.data[key] = { ...this.data[key], ...obj }; + this.setToKv(obj); + }; + + delete = (obj) => { + const key = this.getKey(obj); + delete this.data[key]; + this.deleteFromKv(obj); + }; + + close = () => { + this.state = "closed"; + this.emit(this.state); + this.updateListener?.close(); + this.data = {}; + }; + + public async wait(until: Function, timeout: number = 30): Promise { + if (this.state == "closed") { + throw Error("wait: must not be closed"); + } + return await wait({ + obj: this, + until, + timeout, + change_event: "change-no-throttle", + }); } init = async () => { - this.kv = await getKv({ - nc: this.nc, - project_id: this.project_id, - account_id: this.account_id, + await this.readData(); + }; + + private getKv = reuseInFlight(async () => { + if (this.kv == null) { + this.kv = await getKv({ + nc: this.nc, + project_id: this.project_id, + account_id: this.account_id, + }); + } + return this.kv!; + }); + + // load initial data + private readData = reuseInFlight(async () => { + this.data = await this.getFromKv(); + this.state = "connected"; + this.emit(this.state); + this.listenForUpdates(); + }); + + private listenForUpdates = async () => { + const kv = await this.getKv(); + this.updateListener = await kv.watch({ + key: `${this.natsKeyPrefix}.>`, + // TODO: use this to not have to re-set + // the keys that were set when initializing this + //resumeFromRevision: }); + for await (const { key, value } of this.updateListener) { + const i = key.lastIndexOf("."); + const field = key.slice(i + 1); + const prefix = key.slice(0, i); + if (this.data[prefix] == null) { + this.data[prefix] = {}; + } + const s = this.data[prefix]; + s[field] = this.jc.decode(value); + const changed = [this.primaryString(s)]; + this.emit("change-no-throttle", changed); + this.emit("change", changed); + } }; private primaryString = (obj): string => { @@ -136,29 +233,32 @@ export class SyncTableKV { } }; - set = async (obj) => { + private setToKv = async (obj) => { + const kv = await this.getKv(); const key = this.getKey(obj); for (const field in obj) { const value = this.jc.encode(obj[field]); - await this.kv.put(`${key}.${field}`, value); + await kv.put(`${key}.${field}`, value); } }; - delete = async (obj) => { + private deleteFromKv = async (obj) => { + const kv = await this.getKv(); const key = this.getKey(obj); - const keys = await this.kv.keys(`${key}.>`); + const keys = await kv.keys(`${key}.>`); for await (const k of keys) { - await this.kv.delete(k); + await kv.delete(k); } }; - get = async (obj?, field?) => { + private getFromKv = async (obj?, field?) => { + const kv = await this.getKv(); if (obj == null) { // everything known in this table by the project - const keys = await this.kv.keys(`${this.natsKeyPrefix}.>`); + const keys = await kv.keys(`${this.natsKeyPrefix}.>`); const all: any = {}; for await (const key of keys) { - const mesg = await this.kv.get(key); + const mesg = await kv.get(key); const val = mesg?.sm?.data ? this.jc.decode(mesg.sm.data) : null; if (val != null) { const i = key.lastIndexOf("."); @@ -184,7 +284,7 @@ export class SyncTableKV { // todo: possibly better to just ask for everything under ${key}.> // and take what is needed? Not sure. for (const field of this.fields) { - const mesg = await this.kv.get(`${key}.${field}`); + const mesg = await kv.get(`${key}.${field}`); const val = mesg?.sm?.data ? this.jc.decode(mesg.sm.data) : null; if (val != null) { s[field] = val; @@ -193,7 +293,7 @@ export class SyncTableKV { } return nontrivial ? s : undefined; } - const mesg = await this.kv.get(this.getKey(obj, field)); + const mesg = await kv.get(this.getKey(obj, field)); if (mesg == null) { return undefined; } @@ -202,7 +302,8 @@ export class SyncTableKV { // watch for changes in ONE object async *watchOne(obj) { - const w = await this.kv.watch({ + const kv = await this.getKv(); + const w = await kv.watch({ key: this.getKey(this.getKey(obj), "*"), }); for await (const { key, value } of w) { diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts index 58d7beb68a..64682d6f60 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/nats/sync/synctable-stream.ts @@ -1,5 +1,5 @@ /* -Nats implementation of the idea of a "SyncTable", but +Nats implementation of the idea of a "SyncTable", but for streaming data. This is ONLY for the scope of patches in a single project. @@ -126,7 +126,7 @@ export class SyncTableStream extends EventEmitter { this.consumer = await this.getConsumer(); await this.readData(); this.set_state("connected"); - this.getMessages(); + this.listenForUpdates(); }; private set_state = (state: State): void => { @@ -198,7 +198,7 @@ export class SyncTableStream extends EventEmitter { }; // listen for new data - private getMessages = async () => { + private listenForUpdates = async () => { const consumer = this.consumer!; for await (const mesg of await consumer.consume()) { if (this.handle(mesg, true)) { @@ -242,6 +242,8 @@ export class SyncTableStream extends EventEmitter { // already closed return; } + this.consumer?.close(); + delete this.consumer; this.set_state("closed"); }; From 3f217b979fb16c2d67d6cbe9ab039b63ea9897fd Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 31 Jan 2025 20:23:38 +0000 Subject: [PATCH 079/281] nats synctable: writing complicated code I do NOT like. This feels like a pit of failure. --- src/packages/frontend/client/nats.ts | 15 +++++- src/packages/nats/sync/synctable-kv.ts | 74 +++++++++++++++++++++----- src/packages/nats/sync/synctable.ts | 19 +++++-- 3 files changed, 90 insertions(+), 18 deletions(-) diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 01da3efc72..064525f9cd 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -179,16 +179,27 @@ export class NatsClient { synctable = async ( query, - options?: { obj?: object; atomic?: boolean; stream?: boolean }, + options?: { + obj?: object; + atomic?: boolean; + stream?: boolean; + throttleChanges?: number; + // for tables specific to a project, e.g., syncstrings in a project + project_id?: string; + }, ): Promise => { query = parse_query(query); + const table = keys(query)[0]; const obj = options?.obj; if (obj != null) { - const table = keys(query)[0]; for (const k in obj) { query[table][0][k] = obj[k]; } } + if (options?.project_id != null && query[table][0]["project_id"] === null) { + query[table][0]["project_id"] = options.project_id; + } + console.log(query); const s = createSyncTable({ ...options, query, diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 8f10c7ffc5..be0b80487e 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -16,6 +16,7 @@ import { client_db } from "@cocalc/util/db-schema/client-db"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { EventEmitter } from "events"; import { wait } from "@cocalc/util/async-wait"; +import { throttle } from "lodash"; export function natsKeyPrefix({ query, @@ -82,22 +83,31 @@ export class SyncTableKV extends EventEmitter { private data: { [key: string]: any } = {}; private state: "disconnected" | "connected" | "closed" = "disconnected"; private updateListener?; + private changedKeys: Set = new Set(); + private specifiedByQuery: { [key: string]: any }; constructor({ query, env, account_id, project_id, + throttleChanges = 100, }: { query; env: NatsEnv; account_id?: string; project_id?: string; + throttleChanges?: number; }) { super(); this.sha1 = env.sha1 ?? sha1; this.nc = env.nc; this.jc = env.jc; + this.throttledChangeEvent = throttle( + this.throttledChangeEvent, + throttleChanges, + { leading: false, trailing: true }, + ); const table = keys(query)[0]; this.table = table; this.natsKeyPrefix = natsKeyPrefix({ query, atomic: false }); @@ -105,6 +115,13 @@ export class SyncTableKV extends EventEmitter { this.account_id = account_id ?? query[table][0].account_id; this.primaryKeys = client_db.primary_keys(table); this.primaryKeysSet = new Set(this.primaryKeys); + this.specifiedByQuery = {}; + for (const k in query[table][0]) { + const v = query[table][0][k]; + if (v != null) { + this.specifiedByQuery[k] = v; + } + } this.fields = keys(query[table][0]).filter( (field) => !this.primaryKeysSet.has(field), ); @@ -133,8 +150,9 @@ export class SyncTableKV extends EventEmitter { set = (obj) => { const key = this.getKey(obj); - this.data[key] = { ...this.data[key], ...obj }; - this.setToKv(obj); + const isNew = this.data[key] == null; + this.data[key] = { ...this.data[key], ...obj, ...this.specifiedByQuery }; + this.setToKv(isNew ? { ...obj, ...this.specifiedByQuery } : obj); }; delete = (obj) => { @@ -150,6 +168,10 @@ export class SyncTableKV extends EventEmitter { this.data = {}; }; + get_state = () => { + return this.state; + }; + public async wait(until: Function, timeout: number = 30): Promise { if (this.state == "closed") { throw Error("wait: must not be closed"); @@ -202,18 +224,48 @@ export class SyncTableKV extends EventEmitter { } const s = this.data[prefix]; s[field] = this.jc.decode(value); - const changed = [this.primaryString(s)]; - this.emit("change-no-throttle", changed); - this.emit("change", changed); + + const k = this.primaryString(s); + this.emit("change-no-throttle", [k]); + this.changedKeys.add(k); + this.throttledChangeEvent(); + } + }; + + // this is throttled in constructor + private throttledChangeEvent = () => { + if (this.changedKeys.size > 0) { + this.emit("change", Array.from(this.changedKeys)); + this.changedKeys.clear(); } }; + private fillInFromQuery = (obj) => { + return { ...obj, ...this.specifiedByQuery }; + }; + private primaryString = (obj): string => { if (this.primaryKeys.length === 1) { - return toKey(obj[this.primaryKeys[0]] ?? "")!; + const k = obj[this.primaryKeys[0]]; + if (k == null) { + console.log({ obj }); + throw Error(`primary key '${this.primaryKeys[0]}' not set for object`); + } + return toKey(k)!; } else { // compound primary key - return toKey(this.primaryKeys.map((pk) => obj[pk]))!; + return toKey( + this.primaryKeys.map((pk) => { + const v = obj[pk]; + if (v == null) { + console.log({ obj }); + throw Error( + `part of compound primary key '${pk}' not set for object`, + ); + } + return v; + }), + )!; } }; @@ -221,7 +273,7 @@ export class SyncTableKV extends EventEmitter { if (obj == null) { throw Error("obj must be an object (not null)"); } - return this.sha1(this.primaryString(obj)); + return this.sha1(this.primaryString(this.fillInFromQuery(obj))); }; private getKey = (obj, field?: string): string => { @@ -271,11 +323,7 @@ export class SyncTableKV extends EventEmitter { s[field] = val; } } - const final: any = {}; - for (const k in all) { - final[this.primaryString(all[k])] = all[k]; - } - return final; + return all; } if (field == null) { const s = { ...obj }; diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/nats/sync/synctable.ts index 997591d6f6..7b65c9c3fb 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/nats/sync/synctable.ts @@ -15,6 +15,7 @@ export function createSyncTable({ project_id, atomic, stream, + ...options }: { query; env: NatsEnv; @@ -27,11 +28,23 @@ export function createSyncTable({ if (atomic) { throw Error("atomic stream not implemented yet"); } - return new SyncTableStream({ query, env, account_id, project_id }); + return new SyncTableStream({ + query, + env, + account_id, + project_id, + ...options, + }); } if (atomic) { - return new SyncTableKVAtomic({ query, env, account_id, project_id }); + return new SyncTableKVAtomic({ + query, + env, + account_id, + project_id, + ...options, + }); } else { - return new SyncTableKV({ query, env, account_id, project_id }); + return new SyncTableKV({ query, env, account_id, project_id, ...options }); } } From f6cf4dad587ca57fee25978d46d61ae36088ea42 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 31 Jan 2025 21:13:35 +0000 Subject: [PATCH 080/281] nats kv synctable -- more subtle coding --- src/packages/frontend/client/nats.ts | 1 - src/packages/nats/sync/synctable-kv.ts | 46 +++++++++++++++----------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 064525f9cd..a89727d1bb 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -199,7 +199,6 @@ export class NatsClient { if (options?.project_id != null && query[table][0]["project_id"] === null) { query[table][0]["project_id"] = options.project_id; } - console.log(query); const s = createSyncTable({ ...options, query, diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index be0b80487e..4f054d7922 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -128,6 +128,10 @@ export class SyncTableKV extends EventEmitter { this.readData(); } + init = async () => { + await this.readData(); + }; + get = (obj?) => { if (this.state != "connected") { throw Error("must be connected"); @@ -149,10 +153,10 @@ export class SyncTableKV extends EventEmitter { }; set = (obj) => { + obj = this.fillInFromQuery(obj); const key = this.getKey(obj); - const isNew = this.data[key] == null; - this.data[key] = { ...this.data[key], ...obj, ...this.specifiedByQuery }; - this.setToKv(isNew ? { ...obj, ...this.specifiedByQuery } : obj); + this.data[key] = { ...this.data[key], ...obj }; + this.setToKv(obj); }; delete = (obj) => { @@ -184,10 +188,6 @@ export class SyncTableKV extends EventEmitter { }); } - init = async () => { - await this.readData(); - }; - private getKv = reuseInFlight(async () => { if (this.kv == null) { this.kv = await getKv({ @@ -211,24 +211,31 @@ export class SyncTableKV extends EventEmitter { const kv = await this.getKv(); this.updateListener = await kv.watch({ key: `${this.natsKeyPrefix}.>`, - // TODO: use this to not have to re-set - // the keys that were set when initializing this - //resumeFromRevision: }); - for await (const { key, value } of this.updateListener) { + for await (const { key, value, update } of this.updateListener) { const i = key.lastIndexOf("."); const field = key.slice(i + 1); const prefix = key.slice(0, i); - if (this.data[prefix] == null) { + if (this.data[prefix] == null && value.length > 0) { this.data[prefix] = {}; } const s = this.data[prefix]; - s[field] = this.jc.decode(value); - - const k = this.primaryString(s); - this.emit("change-no-throttle", [k]); - this.changedKeys.add(k); - this.throttledChangeEvent(); + if (update && s != null) { + const k = this.primaryString(s); + this.emit("change-no-throttle", [k]); + this.changedKeys.add(k); + this.throttledChangeEvent(); + } + if (s != null) { + if (value.length == 0 && this.primaryKeysSet.has(field)) { + delete this.data[prefix]; + } else { + s[field] = this.jc.decode(value); + if (Object.keys(s).length == 0) { + delete this.data[prefix]; + } + } + } } }; @@ -301,9 +308,10 @@ export class SyncTableKV extends EventEmitter { for await (const k of keys) { await kv.delete(k); } + await kv.delete(key); }; - private getFromKv = async (obj?, field?) => { + getFromKv = async (obj?, field?) => { const kv = await this.getKv(); if (obj == null) { // everything known in this table by the project From 98dc26b856fec1003214ed6e263a3f39b38615cf Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 31 Jan 2025 22:19:46 +0000 Subject: [PATCH 081/281] nats synctable -- make non-atomic kv synctable work consistently for syncstrings --- src/packages/nats/sync/synctable-kv.ts | 47 +++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 4f054d7922..46fc14c615 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -21,11 +21,26 @@ import { throttle } from "lodash"; export function natsKeyPrefix({ query, atomic = false, + singleton, }: { query; atomic?: boolean; + singleton?: string; }) { - return sha1(jsonStableStringify({ query, atomic })); + if (atomic) { + if (singleton) { + throw Error("not implemented"); + } + return sha1(jsonStableStringify({ query, atomic })); + } else { + // for non-atomic there's no problem with many different queries with the same primary keys. + // we thus just use the table's name + let prefix = keys(query)[0]; + if (singleton) { + prefix += "." + singleton; + } + return prefix; + } } export async function getKv({ @@ -68,6 +83,18 @@ export function toKey(x): string | undefined { } } +function isSingletonQuery(query) { + const table = keys(query)[0]; + const pattern = query[table][0]; + for (const key of client_db.primary_keys(table)) { + if (pattern[key] !== null) { + // a primary key is specified, so there can be only one match + return true; + } + } + return false; +} + export class SyncTableKV extends EventEmitter { private kv?; private nc; @@ -85,6 +112,7 @@ export class SyncTableKV extends EventEmitter { private updateListener?; private changedKeys: Set = new Set(); private specifiedByQuery: { [key: string]: any }; + private singleton?: string; constructor({ query, @@ -110,11 +138,18 @@ export class SyncTableKV extends EventEmitter { ); const table = keys(query)[0]; this.table = table; - this.natsKeyPrefix = natsKeyPrefix({ query, atomic: false }); - this.project_id = project_id ?? query[table][0].project_id; - this.account_id = account_id ?? query[table][0].account_id; this.primaryKeys = client_db.primary_keys(table); this.primaryKeysSet = new Set(this.primaryKeys); + this.project_id = project_id ?? query[table][0].project_id; + this.account_id = account_id ?? query[table][0].account_id; + this.singleton = isSingletonQuery(query) + ? this.natObjectKey(query[table][0]) + : undefined; + this.natsKeyPrefix = natsKeyPrefix({ + query, + atomic: false, + singleton: this.singleton, + }); this.specifiedByQuery = {}; for (const k in query[table][0]) { const v = query[table][0][k]; @@ -284,7 +319,9 @@ export class SyncTableKV extends EventEmitter { }; private getKey = (obj, field?: string): string => { - const x = `${this.natsKeyPrefix}.${this.natObjectKey(obj)}`; + const x = this.singleton + ? this.natsKeyPrefix + : `${this.natsKeyPrefix}.${this.natObjectKey(obj)}`; if (field == null) { return x; } else { From 6e75adc2b7a2fdd3ea2553bc6dc6a31773451fc1 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 31 Jan 2025 22:49:05 +0000 Subject: [PATCH 082/281] nats: plug in new synctable-kv to syncdoc for syncstrings table --- src/packages/nats/package.json | 15 +++++++++--- src/packages/nats/sync/synctable-kv.ts | 21 +++++++++++++--- src/packages/nats/sync/synctable.ts | 5 ++-- src/packages/pnpm-lock.yaml | 3 +++ src/packages/sync/editor/generic/sync-doc.ts | 25 +++++++++++++------- 5 files changed, 52 insertions(+), 17 deletions(-) diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 5ea256b56d..1b2d46bad7 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -16,18 +16,27 @@ "test": "pnpm exec jest", "prepublishOnly": "pnpm test" }, - "files": ["dist/**", "README.md", "package.json"], + "files": [ + "dist/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["utilities", "nats", "cocalc"], + "keywords": [ + "utilities", + "nats", + "cocalc" + ], "license": "SEE LICENSE.md", "dependencies": { + "@cocalc/comm": "workspace:*", "@cocalc/nats": "workspace:*", "@cocalc/util": "workspace:*", - "@cocalc/comm": "workspace:*", "@nats-io/jetstream": "3.0.0-36", "@nats-io/kv": "3.0.0-30", "awaiting": "^3.0.0", "events": "3.3.0", + "immutable": "^4.3.0", "json-stable-stringify": "^1.0.1", "lodash": "^4.17.21", "sha1": "^1.1.1" diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 46fc14c615..c07a53a7da 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -17,6 +17,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { EventEmitter } from "events"; import { wait } from "@cocalc/util/async-wait"; import { throttle } from "lodash"; +import { fromJS, Map } from "immutable"; export function natsKeyPrefix({ query, @@ -113,6 +114,7 @@ export class SyncTableKV extends EventEmitter { private changedKeys: Set = new Set(); private specifiedByQuery: { [key: string]: any }; private singleton?: string; + private getHook: Function; constructor({ query, @@ -120,14 +122,17 @@ export class SyncTableKV extends EventEmitter { account_id, project_id, throttleChanges = 100, + immutable, }: { query; env: NatsEnv; account_id?: string; project_id?: string; throttleChanges?: number; + immutable?: boolean; }) { super(); + this.getHook = immutable ? fromJS : (x) => x; this.sha1 = env.sha1 ?? sha1; this.nc = env.nc; this.jc = env.jc; @@ -176,25 +181,35 @@ export class SyncTableKV extends EventEmitter { for (const k in this.data) { result[this.primaryString(this.data[k])] = this.data[k]; } - return result; + return this.getHook(result); } - return this.data[this.getKey(obj)]; + return this.getHook(this.data[this.getKey(obj)]); }; get_one = () => { for (const key in this.data) { - return this.data[key]; + return this.getHook(this.data[key]); } }; set = (obj) => { + if (Map.isMap(obj)) { + obj = obj.toJS(); + } obj = this.fillInFromQuery(obj); const key = this.getKey(obj); this.data[key] = { ...this.data[key], ...obj }; this.setToKv(obj); }; + save = async () => { + // TODO -- right now it is instantly saving on any change... + } + delete = (obj) => { + if (Map.isMap(obj)) { + obj = obj.toJS(); + } const key = this.getKey(obj); delete this.data[key]; this.deleteFromKv(obj); diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/nats/sync/synctable.ts index 7b65c9c3fb..7743db7bc0 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/nats/sync/synctable.ts @@ -23,10 +23,11 @@ export function createSyncTable({ project_id?: string; atomic?: boolean; stream?: boolean; + immutable?: boolean; // if true for SyncTableKVAtomic, then get/get_one output immutable.js objects }) { if (stream) { - if (atomic) { - throw Error("atomic stream not implemented yet"); + if (!atomic) { + throw Error("non-atomic stream not implemented yet"); } return new SyncTableStream({ query, diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 3e42d11f81..c7be875583 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1072,6 +1072,9 @@ importers: events: specifier: 3.3.0 version: 3.3.0 + immutable: + specifier: ^4.3.0 + version: 4.3.7 json-stable-stringify: specifier: ^1.0.1 version: 1.1.1 diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 06fd0e9c24..a6496f28b6 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1236,11 +1236,11 @@ export class SyncDoc extends EventEmitter { dbg("wrote syncstring to db - done."); } - private async synctable( + private synctable = async ( query, options: any[], throttle_changes?: undefined | number, - ): Promise { + ): Promise => { this.assert_not_closed("synctable"); const dbg = this.dbg("synctable"); if (!this.ephemeral && this.persistent && this.data_server == "project") { @@ -1259,6 +1259,17 @@ export class SyncDoc extends EventEmitter { path: this.path, }, stream: true, + atomic: true, + }); + } else if (this.useNats && query.syncstrings) { + synctable = await this.client.synctable_nats(query, { + obj: { + project_id: this.project_id, + path: this.path, + }, + stream: false, + atomic: false, + immutable: true, }); } else { switch (this.data_server) { @@ -1290,7 +1301,7 @@ export class SyncDoc extends EventEmitter { // will crash the entire process. synctable.on("error", (error) => dbg("ERROR", error)); return synctable; - } + }; private async init_syncstring_table(): Promise { const query = { @@ -1801,11 +1812,7 @@ export class SyncDoc extends EventEmitter { return; } dbg("creating the evaluator and waiting for init"); - this.evaluator = new Evaluator( - this, - this.client, - this.synctable.bind(this), - ); + this.evaluator = new Evaluator(this, this.client, this.synctable); await this.evaluator.init(); dbg("done"); } @@ -1821,7 +1828,7 @@ export class SyncDoc extends EventEmitter { this.ipywidgets_state = new IpywidgetsState( this, this.client, - this.synctable.bind(this), + this.synctable, ); await this.ipywidgets_state.init(); dbg("done"); From cf77d7c734c2967d055029c5ab3d479333344ad0 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 31 Jan 2025 23:09:18 +0000 Subject: [PATCH 083/281] nats sync: get saving to disk to work --- src/packages/sync/editor/generic/sync-doc.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index a6496f28b6..38898b02a6 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -110,6 +110,7 @@ import { } from "./types"; import { patch_cmp } from "./util"; import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; +import mergeDeep from "@cocalc/util/immutable-deep-merge"; export type State = "init" | "ready" | "closed"; export type DataServer = "project" | "database"; @@ -1208,6 +1209,10 @@ export class SyncDoc extends EventEmitter { // patches table uses the string_id, which is a SHA1 hash. private async ensure_syncstring_exists_in_db(): Promise { const dbg = this.dbg("ensure_syncstring_exists_in_db"); + if (this.useNats) { + dbg("skipping -- no database"); + return; + } if (!this.client.is_connected()) { dbg("wait until connected...", this.client.is_connected()); @@ -3379,11 +3384,8 @@ export class SyncDoc extends EventEmitter { emit_change_debounced = debounce(this.emit_change.bind(this), 0); private set_syncstring_table = async (obj, save = true) => { - let value = this.syncstring_table_get_one(); - const value0 = value; - for (const key in obj) { - value = value.set(key, obj[key]); - } + const value0 = this.syncstring_table_get_one(); + const value = mergeDeep(value0, fromJS(obj)); if (value0.equals(value)) { return; } From db5011ee105ef10395c1c93856412d6649315816 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 1 Feb 2025 02:36:25 +0000 Subject: [PATCH 084/281] nats: incorporate open-files service into project so it automatically starts (and can also be remotely terminated) --- src/packages/frontend/client/nats.ts | 75 +++++++++++-------- src/packages/nats/sync/open-files.ts | 2 +- src/packages/nats/sync/synctable-kv-atomic.ts | 25 ++++++- src/packages/nats/sync/synctable-kv.ts | 3 +- src/packages/nats/sync/synctable-stream.ts | 5 +- src/packages/project/client.ts | 4 + src/packages/project/nats/api/index.ts | 26 +++++-- src/packages/project/nats/index.ts | 5 +- src/packages/project/nats/open-files.ts | 36 +++++++-- src/packages/project/nats/synctable.ts | 3 + src/packages/sync/editor/generic/sync-doc.ts | 2 +- 11 files changed, 135 insertions(+), 51 deletions(-) diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index a89727d1bb..dc05b5b966 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -177,37 +177,52 @@ export class NatsClient { }; }; - synctable = async ( - query, - options?: { - obj?: object; - atomic?: boolean; - stream?: boolean; - throttleChanges?: number; - // for tables specific to a project, e.g., syncstrings in a project - project_id?: string; - }, - ): Promise => { - query = parse_query(query); - const table = keys(query)[0]; - const obj = options?.obj; - if (obj != null) { - for (const k in obj) { - query[table][0][k] = obj[k]; - } - } - if (options?.project_id != null && query[table][0]["project_id"] === null) { - query[table][0]["project_id"] = options.project_id; - } - const s = createSyncTable({ - ...options, + private synctableCache: { [key: string]: SyncTable } = {}; + synctable = reuseInFlight( + async ( query, - env: await this.getEnv(), - account_id: this.client.account_id, - }); - await s.init(); - return s; - }; + options?: { + obj?: object; + atomic?: boolean; + stream?: boolean; + throttleChanges?: number; + // for tables specific to a project, e.g., syncstrings in a project + project_id?: string; + }, + ): Promise => { + query = parse_query(query); + const key = JSON.stringify(query); + if (this.synctableCache[key] != null) { + return this.synctableCache[key]; + } + const table = keys(query)[0]; + const obj = options?.obj; + if (obj != null) { + for (const k in obj) { + query[table][0][k] = obj[k]; + } + } + if ( + options?.project_id != null && + query[table][0]["project_id"] === null + ) { + query[table][0]["project_id"] = options.project_id; + } + const s = createSyncTable({ + ...options, + query, + env: await this.getEnv(), + account_id: this.client.account_id, + }); + this.synctableCache[key] = s; + // @ts-ignore + s.on("closed", () => { + delete this.synctableCache[key]; + }); + await s.init(); + return s; + }, + ); changefeedInterest = async (query, noError?: boolean) => { // express interest diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index c2c3b59058..70aff29997 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -91,7 +91,7 @@ export class OpenFiles { close = () => { this.state = "closed"; for (const w of this.watches) { - w.close(); + w.stop(); this.watches = []; } }; diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index 20aca0ff4c..07d6a1d120 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -2,8 +2,10 @@ import { keys } from "lodash"; import { client_db } from "@cocalc/util/db-schema/client-db"; import { getKv, toKey, type NatsEnv, natsKeyPrefix } from "./synctable-kv"; import { sha1 } from "@cocalc/util/misc"; +import { EventEmitter } from "events"; +export type State = "disconnected" | "connected" | "closed"; -export class SyncTableKVAtomic { +export class SyncTableKVAtomic extends EventEmitter { private kv?; private nc; private jc; @@ -13,6 +15,7 @@ export class SyncTableKVAtomic { private primaryKeys: string[]; private project_id?: string; private account_id?: string; + private state: State = "disconnected"; constructor({ query, @@ -25,6 +28,7 @@ export class SyncTableKVAtomic { account_id?: string; project_id?: string; }) { + super(); this.sha1 = env.sha1 ?? sha1; this.nc = env.nc; this.jc = env.jc; @@ -36,12 +40,22 @@ export class SyncTableKVAtomic { this.primaryKeys = client_db.primary_keys(table); } + private set_state = (state: State): void => { + this.state = state; + this.emit(state); + }; + + get_state = () => { + return this.state; + }; + init = async () => { this.kv = await getKv({ nc: this.nc, project_id: this.project_id, account_id: this.account_id, }); + this.set_state("connected"); }; private primaryString = (obj): string => { @@ -98,7 +112,16 @@ export class SyncTableKVAtomic { key: `${this.natsKeyPrefix}.>`, }); for await (const { value } of w) { + if (this.state == "closed") { + return; + } yield this.jc.decode(value); } } + + close = () => { + this.set_state("closed"); + this.removeAllListeners(); + // TODO: stop watchers... ? + }; } diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index c07a53a7da..33c99ed254 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -218,7 +218,8 @@ export class SyncTableKV extends EventEmitter { close = () => { this.state = "closed"; this.emit(this.state); - this.updateListener?.close(); + this.removeAllListeners(); + this.updateListener?.stop(); this.data = {}; }; diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts index 64682d6f60..f2100007c2 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/nats/sync/synctable-stream.ts @@ -242,9 +242,10 @@ export class SyncTableStream extends EventEmitter { // already closed return; } - this.consumer?.close(); - delete this.consumer; this.set_state("closed"); + this.removeAllListeners(); + this.consumer?.delete(); + delete this.consumer; }; delete = async (_obj) => { diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index 76f5b7ea0a..60c60d265d 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -139,6 +139,10 @@ export class Client extends EventEmitter implements ProjectClientInterface { throw Error("BUG: Client already created!"); } ALREADY_CREATED = true; + if (process.env.HOME != null) { + // client assumes curdir is HOME + process.chdir(process.env.HOME); + } this.project_id = data.project_id; this.dbg("constructor")(); this.setMaxListeners(300); // every open file/table/sync db listens for connect event, which adds up. diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index 7dbe377f1c..e42a250df7 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -3,7 +3,7 @@ How to do development (so in a dev project doing cc-in-cc dev). 0. From the browser, terminate this api server running in the project already, if any - await cc.client.nats_client.projectApi({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}).terminate() + await cc.client.nats_client.projectApi({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}).system.terminate({service:'api'}) 1. Open a terminal in the project itself, which sets up the required environment variables, e.g., @@ -38,6 +38,7 @@ import getLogger from "@cocalc/backend/logger"; import { type ProjectApi } from "@cocalc/nats/project-api"; import getConnection from "@cocalc/project/nats/connection"; import { getSubject } from "../names"; +import { terminate as terminateOpenFiles } from "@cocalc/project/nats/open-files"; const logger = getLogger("project:nats:api"); const jc = JSONCodec(); @@ -56,12 +57,23 @@ async function listen(subscription, subject) { const request = jc.decode(mesg.data) ?? ({} as any); // logger.debug("got message", request); if (request.name == "system.terminate") { - // special hook so admin can terminate handling. This is useful for development. - console.warn("TERMINATING listening on ", subject); - logger.debug("TERMINATING listening on ", subject); - mesg.respond(jc.encode({ status: "terminating" })); - subscription.close(); - return; + // TODO: should be part of handleApiRequest below, but done differently because + // one case halts this loop + const { service } = request.args[0] ?? {}; + if (service == "open-files") { + terminateOpenFiles(); + mesg.respond(jc.encode({ status: "terminating", service })); + continue; + } else if (service == "api") { + // special hook so admin can terminate handling. This is useful for development. + console.warn("TERMINATING listening on ", subject); + logger.debug("TERMINATING listening on ", subject); + mesg.respond(jc.encode({ status: "terminating", service })); + subscription.close(); + return; + } else { + mesg.respond(jc.encode({ error: `Unknown service ${service}` })); + } } handleApiRequest(request, mesg); } diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts index e3e71e55b4..ab5bda0813 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/nats/index.ts @@ -7,12 +7,15 @@ Start the NATS servers: import { getLogger } from "@cocalc/project/logger"; import { init as initAPI } from "./api"; +import { init as initOpenFiles } from "./open-files"; +// TODO: initWebsocketApi is temporary import { init as initWebsocketApi } from "./browser-websocket-api"; const logger = getLogger("project:nats:index"); export default async function init() { - logger.debug("starting NATS project servers"); + logger.debug("starting NATS project services"); await initAPI(); + await initOpenFiles(); initWebsocketApi(); } diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 5edb602cfb..02a1625716 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -3,6 +3,11 @@ Handle opening files in a project to save/load from disk and also enable compute DEVELOPMENT: +0. From the browser, terminate this api server running in the project already, if any + + await cc.client.nats_client.projectApi({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}).system.terminate({service:'open-files'}) + + Set env variables as in a project, then: > require("@cocalc/project/nats/open-files").init() @@ -14,7 +19,6 @@ import { compute_server_id, project_id } from "@cocalc/project/data"; import { getEnv } from "./env"; import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; import { getClient } from "@cocalc/project/client"; -//import { SyncDB } from "@cocalc/sync/editor/db/sync"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import getLogger from "@cocalc/backend/logger"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; @@ -22,17 +26,35 @@ import { delay } from "awaiting"; const logger = getLogger("project:nats:open-files"); +let openFiles: OpenFiles | null = null; + export async function init() { logger.debug("init"); - const openFiles = new OpenFiles({ + openFiles = new OpenFiles({ project_id, env: await getEnv(), }); - const entries: { [path: string]: Entry } = {}; - closeIgnoredFiles(entries, openFiles); - for await (const entry of await openFiles.watch()) { - entries[entry.path] = entry; - await handleEntry(entry); + runLoop(); +} + +async function runLoop() { + logger.debug("starting run loop"); + if (openFiles != null) { + const entries: { [path: string]: Entry } = {}; + closeIgnoredFiles(entries, openFiles); + for await (const entry of await openFiles.watch()) { + entries[entry.path] = entry; + await handleEntry(entry); + } + } + logger.debug("exiting open files run loop"); +} + +export function terminate() { + logger.debug("terminating open-files service"); + openFiles?.close(); + for (const path in openSyncDocs) { + closeSyncDoc(path); } } diff --git a/src/packages/project/nats/synctable.ts b/src/packages/project/nats/synctable.ts index 339be82efc..51afd8412c 100644 --- a/src/packages/project/nats/synctable.ts +++ b/src/packages/project/nats/synctable.ts @@ -31,6 +31,9 @@ const synctable = reuseInFlight( }); await s.init(); cache[key] = s; + s.on("closed", () => { + delete cache[key]; + }); } return cache[key]; }, diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 38898b02a6..1637b8e48d 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1146,7 +1146,7 @@ export class SyncDoc extends EventEmitter { await this.async_close(); dbg("async_close -- successfully saved all data to database"); } catch (err) { - dbg("async_close -- ERROR -- ", err); + dbg(`async_close -- ERROR -- ${err}`); } // this avoids memory leaks: close(this); From 2936021e84cfd797d7abd696f9c405c7dcfa045d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 1 Feb 2025 04:15:23 +0000 Subject: [PATCH 085/281] nats sync: backend syncdocs working with proper doctype --- src/packages/nats/sync/synctable-kv.ts | 30 +++++-- src/packages/project/nats/open-files.ts | 86 ++++++++++++++++++-- src/packages/sync/editor/generic/sync-doc.ts | 28 +++++-- 3 files changed, 120 insertions(+), 24 deletions(-) diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 33c99ed254..c77a67b832 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -103,6 +103,7 @@ export class SyncTableKV extends EventEmitter { private sha1; public readonly table; public readonly natsKeyPrefix; + private allFields: Set; private primaryKeys: string[]; private primaryKeysSet: Set; private fields: string[]; @@ -143,6 +144,7 @@ export class SyncTableKV extends EventEmitter { ); const table = keys(query)[0]; this.table = table; + this.allFields = new Set(Object.keys(query[table][0])); this.primaryKeys = client_db.primary_keys(table); this.primaryKeysSet = new Set(this.primaryKeys); this.project_id = project_id ?? query[table][0].project_id; @@ -204,7 +206,7 @@ export class SyncTableKV extends EventEmitter { save = async () => { // TODO -- right now it is instantly saving on any change... - } + }; delete = (obj) => { if (Map.isMap(obj)) { @@ -271,17 +273,27 @@ export class SyncTableKV extends EventEmitter { this.data[prefix] = {}; } const s = this.data[prefix]; - if (update && s != null) { - const k = this.primaryString(s); - this.emit("change-no-throttle", [k]); - this.changedKeys.add(k); - this.throttledChangeEvent(); + if (update && s != null && Object.keys(s).length > 0) { + let k; + try { + k = this.primaryString(s); + } catch { + // s could be {} or just not enough if filled in to compute key. + k = null; + } + if (k != null) { + this.emit("change-no-throttle", [k]); + this.changedKeys.add(k); + this.throttledChangeEvent(); + } } if (s != null) { if (value.length == 0 && this.primaryKeysSet.has(field)) { delete this.data[prefix]; } else { - s[field] = this.jc.decode(value); + if (this.allFields.has(field)) { + s[field] = this.jc.decode(value); + } if (Object.keys(s).length == 0) { delete this.data[prefix]; } @@ -306,7 +318,6 @@ export class SyncTableKV extends EventEmitter { if (this.primaryKeys.length === 1) { const k = obj[this.primaryKeys[0]]; if (k == null) { - console.log({ obj }); throw Error(`primary key '${this.primaryKeys[0]}' not set for object`); } return toKey(k)!; @@ -376,6 +387,9 @@ export class SyncTableKV extends EventEmitter { if (val != null) { const i = key.lastIndexOf("."); const field = key.slice(i + 1); + if (!this.allFields.has(field)) { + continue; + } const prefix = key.slice(0, i); if (all[prefix] == null) { all[prefix] = {}; diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 02a1625716..234ef1ceba 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -20,9 +20,11 @@ import { getEnv } from "./env"; import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; import { getClient } from "@cocalc/project/client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; +import { SyncDB } from "@cocalc/sync/editor/db/sync"; import getLogger from "@cocalc/backend/logger"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { delay } from "awaiting"; +import { client_db } from "@cocalc/util/db-schema"; const logger = getLogger("project:nats:open-files"); @@ -140,16 +142,84 @@ const closeSyncDoc = reuseInFlight(async (path: string) => { const openSyncDoc = reuseInFlight(async (path: string) => { // todo -- will be async and needs to handle SyncDB and all the config... - logger.debug("open", { path }); + logger.debug("openSyncDoc", { path }); const syncDoc = openSyncDocs[path]; if (syncDoc != null) { - return syncDoc; + return; } const client = getClient(); - openSyncDocs[path] = new SyncString({ - project_id, - path, - client, - }); - return openSyncDocs[path]!; + let x; + try { + const string_id = client_db.sha1(project_id, path); + const syncstrings = await client.synctable_nats( + { syncstrings: [{ string_id, doctype: null }] }, + { + stream: false, + atomic: false, + immutable: false, + }, + ); + x = await getTypeAndOpts(syncstrings); + } catch (err) { + logger.debug(`openSyncDoc failed ${err}`); + return; + } + const { type, opts } = x; + logger.debug("openSyncDoc got", { path, type, opts }); + console.log("openSyncDoc got", { path, type, opts }); + + let doc; + if (type == "string") { + doc = new SyncString({ + ...opts, + project_id, + path, + client, + }); + } else { + doc = new SyncDB({ + ...opts, + project_id, + path, + client, + }); + } + openSyncDocs[path] = doc; + return; }); + +async function getTypeAndOpts( + syncstrings, +): Promise<{ type: string; opts: any }> { + let s = syncstrings.get_one(); + if (s == null) { + await syncstrings.wait(() => { + s = syncstrings.get_one(); + return s != null; + }); + } + console.log("s = ", s); + const opts: any = {}; + let type: string = ""; + + let doctype = s.doctype; + if (doctype != null) { + try { + doctype = JSON.parse(doctype); + } catch { + doctype = {}; + } + if (doctype.opts != null) { + for (const k in doctype.opts) { + opts[k] = doctype.opts[k]; + } + } + type = doctype.type; + } + opts.doctype = doctype; + if (type !== "db" && type !== "string") { + // fallback type + type = "string"; + } + return { type, opts }; +} diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 1637b8e48d..6631a3ea7a 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1392,15 +1392,23 @@ export class SyncDoc extends EventEmitter { this.assert_not_closed( "init_all -- before init patch_list, cursors, evaluator, ipywidgets", ); - await Promise.all([ - this.init_patch_list(), - this.init_cursors(), - this.init_evaluator(), - this.init_ipywidgets(), - ]); - this.assert_not_closed("init_all -- after init patch_list"); + // await Promise.all([ + // this.init_patch_list(), + // this.init_cursors(), + // this.init_evaluator(), + // this.init_ipywidgets(), + // ]); + await this.init_patch_list(); + this.assert_not_closed("init_all -- successful init_patch_list"); + await this.init_cursors(); + this.assert_not_closed("init_all -- successful init_patch_cursors"); + await this.init_evaluator(); + this.assert_not_closed("init_all -- successful init_evaluator"); + await this.init_ipywidgets(); + this.assert_not_closed("init_all -- successful init_ipywidgets"); this.init_table_close_handlers(); + this.assert_not_closed("init_all -- successful init_table_close_handlers"); log("file_use_interval"); this.init_file_use_interval(); @@ -1841,6 +1849,10 @@ export class SyncDoc extends EventEmitter { private async init_cursors(): Promise { const dbg = this.dbg("init_cursors"); + if (this.useNats) { + dbg('skipping for now') + return; + } if (!this.cursors) { dbg("done -- do not care about cursors for this syncdoc."); return; @@ -2489,7 +2501,7 @@ export class SyncDoc extends EventEmitter { } const dbg = this.dbg("handle_syncstring_save_state"); dbg( - `state=${state}; this.syncstring_save_state=${this.syncstring_save_state}; this.state=${state}`, + `state='${state}', this.syncstring_save_state='${this.syncstring_save_state}', this.state='${this.state}'`, ); if ( this.state === "ready" && From bc2d855871f144813362b0216abb3c3197ec4da4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 1 Feb 2025 16:03:28 +0000 Subject: [PATCH 086/281] nats auth: implement function to remove project permissions --- src/packages/server/nats/auth.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 7afb18705a..e5051eb5ba 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -38,6 +38,7 @@ export async function nsc( args: string[], { noAccount }: { noAccount?: boolean } = {}, ) { + // console.log(`nsc ${args.join(" ")}`); return await nsc0(noAccount ? args : [...args, "-a", natsAccountName]); } @@ -233,6 +234,19 @@ export async function addProjectPermission({ account_id, project_id }) { await pushToServer(); } +export async function removeProjectPermission({ account_id, project_id }) { + const name = getNatsUserName({ account_id }); + await nsc([ + "edit", + "signing-key", + "--sk", + name, + "--rm", + `project.${project_id}.>,*.project-${project_id}.>`, + ]); + await pushToServer(); +} + export async function getScopedSigningKey( natsUser: string, ): Promise<{ [key: string]: string[] } | null> { From a18e29fb92c4faddb0e249d95a1dd589f02659e5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 1 Feb 2025 17:33:50 +0000 Subject: [PATCH 087/281] nats: add generic pub/sub with event emitter, which I'm now using to try to implement cursors (not done) --- src/packages/frontend/client/client.ts | 3 + src/packages/frontend/client/nats.ts | 14 +++++ src/packages/nats/names.ts | 15 ++++- src/packages/nats/sync/pubsub.ts | 63 +++++++++++++++++++ src/packages/nats/sync/synctable-kv.ts | 8 +-- src/packages/project/client.ts | 5 ++ src/packages/project/nats/pubsub.ts | 13 ++++ src/packages/sync/editor/generic/sync-doc.ts | 22 +++++-- src/packages/sync/editor/generic/types.ts | 1 + .../sync/editor/string/test/client-test.ts | 3 + 10 files changed, 137 insertions(+), 10 deletions(-) create mode 100644 src/packages/nats/sync/pubsub.ts create mode 100644 src/packages/project/nats/pubsub.ts diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 84d9dbdd64..20966c6502 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -77,6 +77,7 @@ export interface WebappClient extends EventEmitter { is_signed_in: () => boolean; synctable_project: Function; synctable_nats: Function; + pubsub_nats: Function; project_websocket: Function; prettier: Function; exec: Function; @@ -159,6 +160,7 @@ class Client extends EventEmitter implements WebappClient { is_signed_in: () => boolean; synctable_project: Function; synctable_nats: Function; + pubsub_nats: Function; project_websocket: Function; prettier: Function; exec: Function; @@ -261,6 +263,7 @@ class Client extends EventEmitter implements WebappClient { this.sync_client, ); this.synctable_nats = this.nats_client.synctable; + this.pubsub_nats = this.nats_client.pubsub; this.query = this.query_client.query.bind(this.query_client); this.async_query = this.query_client.query.bind(this.query_client); diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index dc05b5b966..a4cd4a8587 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -15,6 +15,7 @@ import { type ProjectApi, initProjectApi } from "@cocalc/nats/project-api"; import { getPrimusConnection } from "@cocalc/nats/primus"; import { isValidUUID } from "@cocalc/util/misc"; import { OpenFiles } from "@cocalc/nats/sync/open-files"; +import { PubSub } from "@cocalc/nats/sync/pubsub"; export class NatsClient { /*private*/ client: WebappClient; @@ -185,6 +186,7 @@ export class NatsClient { obj?: object; atomic?: boolean; stream?: boolean; + pubsub?: boolean; throttleChanges?: number; // for tables specific to a project, e.g., syncstrings in a project project_id?: string; @@ -267,4 +269,16 @@ export class NatsClient { } return this.openFilesCache[project_id]!; }); + + pubsub = async ({ + project_id, + path, + name, + }: { + project_id: string; + path?: string; + name: string; + }) => { + return new PubSub({ project_id, path, name, env: await this.getEnv() }); + }; } diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts index 7bb637705b..d9372e1a4f 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/nats/names.ts @@ -1,3 +1,16 @@ +/* +Names we use with nats. + +For Jetstream: + +project-{project_id}-{compute_server_id}[.-service][.-sha1(path)] + +For Subjects: + + project.{project-id}.{compute_server_id}[.{service}][.{path}] + +*/ + import { sha1 } from "@cocalc/util/misc"; import generateVouchers from "@cocalc/util/vouchers"; @@ -7,7 +20,7 @@ export function randomId() { return generateVouchers({ count: 1, length: 10 })[0]; } -// project-{project_id}-{compute_server_id}[.-service][.-sha1(path)] + export function projectSubject({ project_id, diff --git a/src/packages/nats/sync/pubsub.ts b/src/packages/nats/sync/pubsub.ts new file mode 100644 index 0000000000..7b782c8508 --- /dev/null +++ b/src/packages/nats/sync/pubsub.ts @@ -0,0 +1,63 @@ +/* +Use NATS simple pub/sub to share state for something *ephemeral* in a project. +*/ + +import { projectSubject } from "@cocalc/nats/names"; +import { type NatsEnv } from "./synctable-kv"; +import { EventEmitter } from "events"; +import { State } from "./synctable-kv-atomic"; + +export class PubSub extends EventEmitter { + private subject: string; + private env: NatsEnv; + private sub?; + private state: State = "disconnected"; + + constructor({ + project_id, + path, + name, + env, + }: { + project_id: string; + name: string; + path?: string; + env: NatsEnv; + }) { + super(); + this.env = env; + this.subject = projectSubject({ + project_id, + path, + service: `pubsub-${name}`, + }); + this.subscribe(); + } + + private setState = (state: State) => { + this.state = state; + this.emit(state); + }; + + close = () => { + if (this.state == "closed") { + return; + } + this.setState("closed"); + this.removeAllListeners(); + this.sub?.close(); + delete this.sub; + }; + + set = (obj) => { + this.env.nc.publish(this.subject, this.env.jc.encode(obj)); + }; + + private subscribe = async () => { + this.sub = this.env.nc.subscribe(this.subject); + this.setState("connected"); + for await (const mesg of this.sub) { + this.emit("change", this.env.jc.decode(mesg.data)); + } + }; +} diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index c77a67b832..ba5e706c3f 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -88,12 +88,12 @@ function isSingletonQuery(query) { const table = keys(query)[0]; const pattern = query[table][0]; for (const key of client_db.primary_keys(table)) { - if (pattern[key] !== null) { - // a primary key is specified, so there can be only one match - return true; + if (pattern[key] === null) { + // part of primary key is NOT specified, so not singleton + return false; } } - return false; + return true; } export class SyncTableKV extends EventEmitter { diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index 60c60d265d..4fc7685c00 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -49,6 +49,7 @@ import { getListingsTable } from "@cocalc/project/sync/listings"; import { get_synctable } from "./sync/open-synctables"; import { get_syncdoc } from "./sync/sync-doc"; import synctable_nats from "@cocalc/project/nats/synctable"; +import pubsub from "@cocalc/project/nats/pubsub"; const winston = getLogger("client"); @@ -510,6 +511,10 @@ export class Client extends EventEmitter implements ProjectClientInterface { return await synctable_nats(query, options); }; + pubsub_nats = async ({ path, name }: { path?: string; name: string }) => { + return await pubsub({ path, name }); + }; + // WARNING: making two of the exact same sync_string or sync_db will definitely // lead to corruption! diff --git a/src/packages/project/nats/pubsub.ts b/src/packages/project/nats/pubsub.ts new file mode 100644 index 0000000000..e02a26a3ab --- /dev/null +++ b/src/packages/project/nats/pubsub.ts @@ -0,0 +1,13 @@ +import { getEnv } from "./env"; +import { PubSub } from "@cocalc/nats/sync/pubsub"; +import { project_id } from "@cocalc/project/data"; + +export default async function pubsub({ + path, + name, +}: { + path?: string; + name: string; +}) { + return new PubSub({ env: await getEnv(), project_id, path, name }); +} diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 6631a3ea7a..fae5813462 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -600,6 +600,10 @@ export class SyncDoc extends EventEmitter { // will actually always be non-null due to above this.cursor_last_time = x.time; } + if (this.useNats) { + this.cursors_table.set(x); + return; + } this.cursors_table.set(x, "none"); await this.cursors_table.save(); }; @@ -1059,7 +1063,7 @@ export class SyncDoc extends EventEmitter { for (const x of ["syncstring", "patches", "cursors"]) { const t = this[x + "_table"]; if (t != null) { - t.on("close", () => this.close()); + t.on("close", this.close); } } } @@ -1849,14 +1853,22 @@ export class SyncDoc extends EventEmitter { private async init_cursors(): Promise { const dbg = this.dbg("init_cursors"); - if (this.useNats) { - dbg('skipping for now') - return; - } if (!this.cursors) { dbg("done -- do not care about cursors for this syncdoc."); return; } + if (this.useNats) { + dbg("NATS cursors support using pub/sub"); + this.cursors_table = await this.client.synctable_nats({ + project_id: this.project_id, + path: this.path, + name: "cursors", + pubsub: true, + }); + + return; + } + dbg("getting cursors ephemeral table"); const query = { cursors: [ diff --git a/src/packages/sync/editor/generic/types.ts b/src/packages/sync/editor/generic/types.ts index 942235ac88..44e3039d13 100644 --- a/src/packages/sync/editor/generic/types.ts +++ b/src/packages/sync/editor/generic/types.ts @@ -96,6 +96,7 @@ export interface ProjectClient extends EventEmitter { ) => Promise; synctable_nats: (query: any, obj?) => Promise; + pubsub_nats: (query: any, obj?) => Promise; // account_id or project_id or compute_server_id (encoded as a UUID - use decodeUUIDtoNum to decode) client_id: () => string; diff --git a/src/packages/sync/editor/string/test/client-test.ts b/src/packages/sync/editor/string/test/client-test.ts index 49e3d13ad4..798f3a8d5f 100644 --- a/src/packages/sync/editor/string/test/client-test.ts +++ b/src/packages/sync/editor/string/test/client-test.ts @@ -174,6 +174,9 @@ export class Client extends EventEmitter implements Client0 { async synctable_nats(_query: any): Promise { throw Error("not implemented"); } + async pubsub_nats(_query: any): Promise { + throw Error("not implemented"); + } // account_id or project_id public client_id(): string { From a72862d3ad9118a44e46b4de15135159ed3da1af Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 00:02:54 +0000 Subject: [PATCH 088/281] nats cursors -- make it work :-) --- src/packages/sync/editor/generic/sync-doc.ts | 71 +++++++++++++++----- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index fae5813462..a6cf79352b 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -55,6 +55,9 @@ const READ_ONLY_CHECK_INTERVAL_MS = 7500; // cursors less responsive. const CURSOR_THROTTLE_MS = 750; +// NATS is much faster and can handle load, and cursors only uses pub/sub +const CURSOR_THROTTLE_NATS_MS = 150; + // Ignore file changes for this long after save to disk. const RECENT_SAVE_TO_DISK_MS = 2000; @@ -580,8 +583,25 @@ export class SyncDoc extends EventEmitter { // table not initialized yet return; } + if (this.useNats) { + const time = this.client.server_time().valueOf(); + const x: { + user_id: number; + locs: any; + time: number; + } = { + user_id: this.my_user_id, + locs, + time, + }; + // will actually always be non-null due to above + this.cursor_last_time = new Date(x.time); + this.cursors_table.set(x); + return; + } + const x: { - string_id: string; + string_id?: string; user_id: number; locs: any[]; time?: Date; @@ -600,18 +620,18 @@ export class SyncDoc extends EventEmitter { // will actually always be non-null due to above this.cursor_last_time = x.time; } - if (this.useNats) { - this.cursors_table.set(x); - return; - } this.cursors_table.set(x, "none"); await this.cursors_table.save(); }; - set_cursor_locs = throttle(this.setCursorLocsNoThrottle, CURSOR_THROTTLE_MS, { - leading: true, - trailing: true, - }); + set_cursor_locs = throttle( + this.setCursorLocsNoThrottle, + USE_NATS ? CURSOR_THROTTLE_NATS_MS : CURSOR_THROTTLE_MS, + { + leading: true, + trailing: true, + }, + ); private init_file_use_interval(): void { if (this.file_use_interval == null) { @@ -1859,13 +1879,32 @@ export class SyncDoc extends EventEmitter { } if (this.useNats) { dbg("NATS cursors support using pub/sub"); - this.cursors_table = await this.client.synctable_nats({ + this.cursors_table = await this.client.pubsub_nats({ project_id: this.project_id, path: this.path, name: "cursors", - pubsub: true, }); - + this.cursors_table.on( + "change", + (obj: { user_id: number; locs: any; time: number }) => { + const account_id = this.users[obj.user_id]; + if (!account_id) { + return; + } + if (obj.locs == null && !this.cursor_map.has(account_id)) { + // gone, and already gone. + return; + } + if (obj.locs != null) { + // changed + this.cursor_map = this.cursor_map.set(account_id, fromJS(obj)); + } else { + // deleted + this.cursor_map = this.cursor_map.delete(account_id); + } + this.emit("cursor_activity", account_id); + }, + ); return; } @@ -1910,7 +1949,7 @@ export class SyncDoc extends EventEmitter { this.cursor_map = this.cursor_map.set(this.users[u[1]], locs); } }); - this.cursors_table.on("change", this.handle_cursors_change.bind(this)); + this.cursors_table.on("change", this.handle_cursors_change); if (this.cursors_table.setOnDisconnect != null) { // setOnDisconnect is available, so clear our @@ -1928,7 +1967,7 @@ export class SyncDoc extends EventEmitter { dbg("done"); } - private handle_cursors_change(keys): void { + private handle_cursors_change = (keys) => { if (this.state === "closed") { return; } @@ -1957,7 +1996,7 @@ export class SyncDoc extends EventEmitter { } this.emit("cursor_activity", account_id); } - } + }; /* Returns *immutable* Map from account_id to list of cursor positions, if cursors are enabled. @@ -1991,7 +2030,7 @@ export class SyncDoc extends EventEmitter { excludeSelf == "always" || (excludeSelf == "heuristic" && this.cursor_last_time >= - (map.getIn([account_id, "time"], new Date(0)) as Date)) + new Date(map.getIn([account_id, "time"], 0) as number)) ) { map = map.delete(account_id); } From 82ac7c0fd0b315956403e082e3a701de105f6f5f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 01:05:34 +0000 Subject: [PATCH 089/281] enable nats-based synctable for database changefeeds; also make the initial load vastly faster via a "clever trick" --- src/packages/nats/sync/synctable-kv-atomic.ts | 26 ++++++++++++++----- src/packages/nats/util.ts | 10 ++++++- src/packages/sync/table/synctable.ts | 2 +- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index 07d6a1d120..3e72ae3734 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -3,6 +3,7 @@ import { client_db } from "@cocalc/util/db-schema/client-db"; import { getKv, toKey, type NatsEnv, natsKeyPrefix } from "./synctable-kv"; import { sha1 } from "@cocalc/util/misc"; import { EventEmitter } from "events"; +import { numKeys } from "@cocalc/nats/util"; export type State = "disconnected" | "connected" | "closed"; export class SyncTableKVAtomic extends EventEmitter { @@ -94,22 +95,35 @@ export class SyncTableKVAtomic extends EventEmitter { get = async (obj?) => { if (obj == null) { - // everything - const keys = await this.kv.keys(`${this.natsKeyPrefix}.>`); + // get everything. NOte that getting the keys, then the value + // for each key, which is what the JS API docs suggests (?)... is **insanely slow**. + // Instead use watch and stop when we have enough. + // This is *slightly* dangerous, in case the size of the kv store changed maybe right + // after computing the size, but it's a risk we must take. + const key = `${this.natsKeyPrefix}.>`; + const total = await numKeys(this.kv, key); + const watch = await this.kv.watch({ key }); + let count = 0; const all: any = {}; - for await (const key of keys) { - const value = this.decode(await this.kv.get(key)); + for await (const x of watch) { + const value = this.jc.decode(x.value); all[this.primaryString(value)] = value; + count += 1; + if (count >= total) { + break; + } } return all; + } else { + return this.decode(await this.kv.get(this.getKey(obj))); } - return this.decode(await this.kv.get(this.getKey(obj))); }; - // watch for changes + // watch for new changes async *watch() { const w = await this.kv.watch({ key: `${this.natsKeyPrefix}.>`, + include: "updates", }); for await (const { value } of w) { if (this.state == "closed") { diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index 9da2b2be63..35e32ad8e8 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -1,3 +1,11 @@ +// Get the number of keys in a nats kv store, matching a given subject: +export async function numKeys(kv, x: string | string[] = ">"): Promise { + let num = 0; + for await (const _ of await kv.keys(x)) { + num += 1; + } + return num; +} export function handleErrorMessage(mesg) { if (mesg?.error) { @@ -8,4 +16,4 @@ export function handleErrorMessage(mesg) { } } return mesg; -} \ No newline at end of file +} diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index 22e6b610ac..77955e9918 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -19,7 +19,7 @@ let DEBUG: boolean = false; // enable experimental nats database backed changefeed. // for this to work you must explicitly run the server in @cocalc/database/nats -const USE_NATS = false; +const USE_NATS = true; export function set_debug(x: boolean): void { DEBUG = x; From 5ea6e0c1a67b5a96d53abf2d8c809837dcdd6394 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 02:03:34 +0000 Subject: [PATCH 090/281] nats: integrate hub database service; refactor and use getAll more --- src/packages/database/nats/index.ts | 35 +++++++++---- src/packages/nats/hub-api/system.ts | 2 + src/packages/nats/sync/synctable-kv-atomic.ts | 23 +++------ src/packages/nats/sync/synctable-kv.ts | 11 ++-- src/packages/nats/util.ts | 50 +++++++++++++++++++ src/packages/project/nats/api/index.ts | 5 +- src/packages/server/nats/api/index.ts | 30 ++++++++--- src/packages/server/nats/api/system.ts | 2 + src/packages/server/nats/index.ts | 2 + src/packages/sync/table/changefeed-nats.ts | 1 - 10 files changed, 121 insertions(+), 40 deletions(-) diff --git a/src/packages/database/nats/index.ts b/src/packages/database/nats/index.ts index e32de47ac0..cd0d3deb01 100644 --- a/src/packages/database/nats/index.ts +++ b/src/packages/database/nats/index.ts @@ -1,5 +1,10 @@ /* +1. turn off nats-server handling for the hub by sending this message from a browser as an admin: + + await cc.client.nats_client.hub.system.terminate({service:'database'}) + +2. Run this echo "require('@cocalc/database/nats').init()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node @@ -9,7 +14,7 @@ import getLogger from "@cocalc/backend/logger"; import { JSONCodec } from "nats"; import userQuery from "@cocalc/database/user-query"; import { getConnection } from "@cocalc/backend/nats"; -import { getUserId } from "@cocalc/nats/api"; +import { getUserId } from "@cocalc/nats/hub-api"; import { callback } from "awaiting"; import { db } from "@cocalc/database"; import { @@ -21,25 +26,31 @@ import jsonStableStringify from "json-stable-stringify"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { uuid } from "@cocalc/util/misc"; import { delay } from "awaiting"; +import type { Subscription } from "nats"; const logger = getLogger("database:nats"); const jc = JSONCodec(); +let subscription: Subscription | null = null; export async function init() { const subject = "hub.*.*.db"; logger.debug(`init -- subject='${subject}', options=`, { queue: "0", }); const nc = await getConnection(); - const sub = nc.subscribe(subject, { queue: "0" }); - for await (const mesg of sub) { + subscription = nc.subscribe(subject, { queue: "0" }); + for await (const mesg of subscription) { handleRequest(mesg, nc); } } +export function terminate() { + logger.debug("terminating"); + subscription?.unsubscribe(); +} + async function handleRequest(mesg, nc) { - console.log({ subject: mesg.subject }); let resp; try { const { account_id, project_id } = getUserId(mesg.subject); @@ -47,11 +58,11 @@ async function handleRequest(mesg, nc) { if (!name) { throw Error("api endpoint name must be given in message"); } - logger.debug("handling hub db request:", { + logger.debug("handling database request:", { account_id, project_id, name, - args, + //args, }); resp = await getResponse({ name, args, account_id, project_id, nc }); } catch (err) { @@ -74,6 +85,10 @@ async function getResponse({ name, args, account_id, project_id, nc }) { } } +function queryTable(query) { + return Object.keys(query)[0]; +} + // This is tricky. We return the first result as a normal // async function, but then handle (and don't return) // the subsequent calls to cb generated by the changefeed. @@ -86,10 +101,10 @@ const createChangefeed = reuseInFlight( const now = Date.now(); if (changefeedInterest[hash]) { changefeedInterest[hash] = now; - logger.debug("using existing changefeed for", query); + logger.debug("using existing changefeed for", queryTable(query)); return; } - logger.debug("creating new changefeed for", query); + logger.debug("creating new changefeed for", queryTable(query)); const changes = uuid(); const env = { nc, jc, sha1 }; const synctable = createSyncTable({ @@ -111,13 +126,13 @@ const createChangefeed = reuseInFlight( cb(err); if (result != null) { for (const x of result[synctable.table]) { - logger.debug("changefeed init", x); + // logger.debug("changefeed init", x); await synctable.set(x); } } return; } - logger.debug("changefeed", result); + // logger.debug("changefeed", result); const { action, new_val, old_val } = result as any; // action = 'insert', 'update', 'delete', 'close' // e.g., {"action":"insert","new_val":{"title":"testingxxxxx","project_id":"81e0c408-ac65-4114-bad5-5f4b6539bd0e"}} diff --git a/src/packages/nats/hub-api/system.ts b/src/packages/nats/hub-api/system.ts index deac71fc47..b78aa7b2b8 100644 --- a/src/packages/nats/hub-api/system.ts +++ b/src/packages/nats/hub-api/system.ts @@ -5,10 +5,12 @@ export const system = { getCustomize: noAuth, ping: noAuth, addProjectPermission: authFirst, + terminate: authFirst, }; export interface System { getCustomize: (fields?: string[]) => Promise; ping: () => { now: number }; addProjectPermission: (opts: { project_id: string }) => Promise; + terminate: (service: "database" | "api") => Promise; } diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index 3e72ae3734..32e2da01bc 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -3,7 +3,7 @@ import { client_db } from "@cocalc/util/db-schema/client-db"; import { getKv, toKey, type NatsEnv, natsKeyPrefix } from "./synctable-kv"; import { sha1 } from "@cocalc/util/misc"; import { EventEmitter } from "events"; -import { numKeys } from "@cocalc/nats/util"; +import { getAllFromKv } from "@cocalc/nats/util"; export type State = "disconnected" | "connected" | "closed"; export class SyncTableKVAtomic extends EventEmitter { @@ -95,23 +95,14 @@ export class SyncTableKVAtomic extends EventEmitter { get = async (obj?) => { if (obj == null) { - // get everything. NOte that getting the keys, then the value - // for each key, which is what the JS API docs suggests (?)... is **insanely slow**. - // Instead use watch and stop when we have enough. - // This is *slightly* dangerous, in case the size of the kv store changed maybe right - // after computing the size, but it's a risk we must take. - const key = `${this.natsKeyPrefix}.>`; - const total = await numKeys(this.kv, key); - const watch = await this.kv.watch({ key }); - let count = 0; + const raw = await getAllFromKv({ + kv: this.kv, + key: `${this.natsKeyPrefix}.>`, + }); const all: any = {}; - for await (const x of watch) { - const value = this.jc.decode(x.value); + for (const x of Object.values(raw)) { + const value = this.jc.decode(x); all[this.primaryString(value)] = value; - count += 1; - if (count >= total) { - break; - } } return all; } else { diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index ba5e706c3f..3fff8c2edd 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -18,6 +18,7 @@ import { EventEmitter } from "events"; import { wait } from "@cocalc/util/async-wait"; import { throttle } from "lodash"; import { fromJS, Map } from "immutable"; +import { getAllFromKv } from "@cocalc/nats/util"; export function natsKeyPrefix({ query, @@ -379,12 +380,12 @@ export class SyncTableKV extends EventEmitter { const kv = await this.getKv(); if (obj == null) { // everything known in this table by the project - const keys = await kv.keys(`${this.natsKeyPrefix}.>`); + const raw = await getAllFromKv({ kv, key: `${this.natsKeyPrefix}.>` }); const all: any = {}; - for await (const key of keys) { - const mesg = await kv.get(key); - const val = mesg?.sm?.data ? this.jc.decode(mesg.sm.data) : null; - if (val != null) { + for (const key in raw) { + const x = raw[key]; + if (x) { + const val = this.jc.decode(x); const i = key.lastIndexOf("."); const field = key.slice(i + 1); if (!this.allFields.has(field)) { diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index 35e32ad8e8..10a6108c85 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -7,6 +7,56 @@ export async function numKeys(kv, x: string | string[] = ">"): Promise { return num; } +// get everything from a KV store matching a subject pattern. +// TRICK! Note that getting the keys, then the value +// for each key, which is what the JS API docs suggests (?)... +// is **INSANELY SLOW**! +// Instead count the keys, then use watch and stop when we have them all. +// It's ridiculous but fast, and is *slightly* dangerous, since the size +// of the kv store changed maybe right after computing the size, but +// it's a risk we must take. We put in a 5s default timeout to avoid +// any possibility of hanging forever as a result. +export async function getAllFromKv({ + kv, + key = ">", + timeout = 5000, +}: { + kv; + key?: string | string[]; + timeout?: number; +}): Promise<{ [key: string]: any }> { + const total = await numKeys(kv, key); + let count = 0; + const all: any = {}; + if (total == 0) { + return all; + } + const watch = await kv.watch({ key }); + let id: any = 0; + for await (const { key, value } of watch) { + all[key] = value; + count += 1; + + if (id) { + clearTimeout(id); + id = 0; + } + if (count >= total) { + break; + } + // make a timeout so if the wait from one iteration to the + // next in the loop is more than this amount, it stops. + // This should never happen unless the network were somehow VERY slow + // or the kv size shrunk at the exactly wrong time (and even then, + // it might work due to delete notifications). This is only about + // getting data from NATS, not the database, so should always be fast. + id = setTimeout(() => { + watch.stop(); + }, timeout); + } + return all; +} + export function handleErrorMessage(mesg) { if (mesg?.error) { if (mesg.error.startsWith("Error: ")) { diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index e42a250df7..de6ec3f64a 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -69,13 +69,14 @@ async function listen(subscription, subject) { console.warn("TERMINATING listening on ", subject); logger.debug("TERMINATING listening on ", subject); mesg.respond(jc.encode({ status: "terminating", service })); - subscription.close(); + subscription.unsubscribe(); return; } else { mesg.respond(jc.encode({ error: `Unknown service ${service}` })); } + } else { + handleApiRequest(request, mesg); } - handleApiRequest(request, mesg); } } diff --git a/src/packages/server/nats/api/index.ts b/src/packages/server/nats/api/index.ts index 82f617a060..20c19f100f 100644 --- a/src/packages/server/nats/api/index.ts +++ b/src/packages/server/nats/api/index.ts @@ -5,7 +5,7 @@ To do development: 1. turn off nats-server handling for the hub by sending this message from a browser as an admin: - await cc.client.nats_client.callHub({name:"terminate"}) + await cc.client.nats_client.hub.system.terminate({service:'api'}) 2. Run this script standalone: @@ -39,6 +39,7 @@ import getLogger from "@cocalc/backend/logger"; import { type HubApi, getUserId, transformArgs } from "@cocalc/nats/hub-api"; import { getConnection } from "@cocalc/backend/nats"; import userIsInGroup from "@cocalc/server/accounts/is-in-group"; +import { terminate as terminateDatabase } from "@cocalc/database/nats"; const logger = getLogger("server:nats:api"); @@ -53,16 +54,33 @@ export async function initAPI() { const sub = nc.subscribe(subject, { queue: "0" }); for await (const mesg of sub) { const request = jc.decode(mesg.data) ?? ({} as any); - if (request.name == "terminate") { + if (request.name == "system.terminate") { // special hook so admin can terminate handling. This is useful for development. const { account_id } = getUserId(mesg.subject); if (!(!!account_id && (await userIsInGroup(account_id, "admin")))) { - throw Error("only admin can terminate"); + mesg.respond(jc.encode({ error: "only admin can terminate" })); + continue; } - mesg.respond(jc.encode({ status: "terminating" })); - return; + // TODO: should be part of handleApiRequest below, but done differently because + // one case halts this loop + const { service } = request.args[0] ?? {}; + if (service == "database") { + terminateDatabase(); + mesg.respond(jc.encode({ status: "terminating", service })); + continue; + } else if (service == "api") { + // special hook so admin can terminate handling. This is useful for development. + console.warn("TERMINATING listening on ", subject); + logger.debug("TERMINATING listening on ", subject); + mesg.respond(jc.encode({ status: "terminating", service })); + sub.unsubscribe(); + return; + } else { + mesg.respond(jc.encode({ error: `Unknown service ${service}` })); + } + } else { + handleApiRequest(request, mesg); } - handleApiRequest(request, mesg); } } diff --git a/src/packages/server/nats/api/system.ts b/src/packages/server/nats/api/system.ts index c964eb5b6a..5e0fd7337d 100644 --- a/src/packages/server/nats/api/system.ts +++ b/src/packages/server/nats/api/system.ts @@ -6,3 +6,5 @@ export { addProjectPermission }; export function ping() { return { now: Date.now() }; } + +export async function terminate() {} diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts index d4603354b5..4057d725d8 100644 --- a/src/packages/server/nats/index.ts +++ b/src/packages/server/nats/index.ts @@ -1,5 +1,6 @@ import getLogger from "@cocalc/backend/logger"; import { initAPI } from "./api"; +import { init as initDatabase } from "@cocalc/database/nats"; const logger = getLogger("server:nats"); @@ -7,4 +8,5 @@ export default async function initNatsServer() { logger.debug("initializing nats cocalc hub server"); // do NOT await this! initAPI(); + initDatabase(); } diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index b5e4352abc..e1a05b2d9a 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -19,7 +19,6 @@ export class NatsChangefeed extends EventEmitter { this.client = client; this.query = query; this.options = options; - console.log('changefeed-nats', this.query, this.options); } connect = async () => { From b4567d30d065f8c34a6e946447bbb20a6d438778 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 02:08:38 +0000 Subject: [PATCH 091/281] nats: typescript fix and reminder --- src/packages/sync/table/changefeed-nats.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index e1a05b2d9a..61616405d4 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -19,6 +19,9 @@ export class NatsChangefeed extends EventEmitter { this.client = client; this.query = query; this.options = options; + if (this.options != null && this.options.length > 0) { + console.log("NatsChangefeed -- todo: options not implemented", options); + } } connect = async () => { From 851c7a625962d52d38f707724a61f209eca9f65d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 03:50:18 +0000 Subject: [PATCH 092/281] nats: improve docs and initial configuration --- src/README.md | 18 ++++++++++-------- src/package.json | 4 ++-- src/packages/backend/nats/conf.ts | 24 ++++++++++++++++++++---- src/packages/backend/nats/index.ts | 30 ++++++++++++++++++++---------- 4 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/README.md b/src/README.md index d8af265355..7747789b08 100644 --- a/src/README.md +++ b/src/README.md @@ -1,12 +1,12 @@ # How to Build and Run CoCalc -Updated: **Jan 2023** +**Updated: Feb 2025** -CoCalc is a pretty large and complicated project, and it will only work with the current standard LTS release of node.js \( at least 16.8.x\) and a recent version of [pnpm](https://pnpm.io/). +CoCalc is a pretty large and complicated project, and it will only work with the current standard LTS release of node.js \( at least 18.17.1\) and a recent version of [pnpm](https://pnpm.io/). Also, you will need a LOT of RAM, a minimum of 16 GB. **It's very painful to do development with less than 32 GB of RAM.** **Node.js and NPM Version Requirements:** -- You must be using Node version 16.8.x or newer. **CoCalc will definitely NOT work with any older version!** In a [CoCalc.com](http://CoCalc.com) project, you can put this in `~/.bashrc` to get a valid node version: +- You must be using Node version 18.17.1 or newer. **CoCalc will definitely NOT work with any older version!** In a [CoCalc.com](http://CoCalc.com) project, you can put this in `~/.bashrc` to get a valid node version: ```sh . /cocalc/nvm/nvm.sh @@ -64,14 +64,15 @@ Launch the install and build **for doing development:** these commands via [NPM run scripts](https://docs.npmjs.com/cli/v10/using-npm/scripts). ```sh -~/cocalc/src$ pnpm make-dev +~/cocalc/src$ pnpm build-dev ``` -This will do `pnpm install` for all packages, and also build the typescript/coffeescript, and anything else into a dist directory for each module. Once `pnpm make` finishes successfully, you can start using CoCalc by starting the database and the backend hub in two separate terminals. +This will do `pnpm install` for all packages, and also build the typescript/coffeescript, and anything else into a dist directory for each module. Once `pnpm build-dev` finishes successfully, you can start using CoCalc by starting the database, nats server and the backend hub in three terminals. \(Note that 'pnpm nats\-server' will download, install and configure NATS automatically.\) ```sh -~/cocalc/src$ pnpm database # in one terminal -~/cocalc/src$ pnpm hub # in another terminal +~/cocalc/src$ pnpm database # in one terminal +~/cocalc/src$ pnpm nats-server # in one terminal +~/cocalc/src$ pnpm hub # in another terminal ``` The hub will send minimal logging to stdout, and the rest to `data/logs/log`. @@ -95,7 +96,7 @@ The main \(only?\) difference is that static and next webpack builds are created If necessary, you can delete all the `node_modules` and `dist` directories in all packages and start over as follows: ```sh -~/cocalc/src$ pnpm clean && pnpm make-dev +~/cocalc/src$ pnpm clean && pnpm build-dev ``` ## Doing Development @@ -218,3 +219,4 @@ Regarding VS Code, the relevant settings can be found by searching for "autosave There's some `@cocalc/` packages at [NPMJS.com](http://NPMJS.com). However, _**we're no longer using**_ _**them in any way**_, and don't plan to publish anything new unless there is a compelling use case. + diff --git a/src/package.json b/src/package.json index e64adf3abc..9b91651559 100644 --- a/src/package.json +++ b/src/package.json @@ -18,8 +18,8 @@ "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test", "test": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r test", "prettier-all": "cd packages/", - "nats-server": "cd ${COCALC_ROOT:=$INIT_CWD}/data/nats && ./bin/nats-server -c server.conf", - "nats": "echo 'Starting NATS subshell where you can use: nsc, nats, nats-server'; XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:$PATH PS1=\"nats> \" bash" + "nats-server": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/conf').main()\" && cd ${COCALC_ROOT:=$INIT_CWD}/data/nats && ./bin/nats-server -c server.conf", + "nats": "echo; echo '# Put the following in your ~/.bashrc or use this subshell'; echo; echo \"export XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data\"; echo \"export XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data\"; echo \"export PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:\\$PATH\"; echo; echo; XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:$PATH bash" }, "repository": { "type": "git", diff --git a/src/packages/backend/nats/conf.ts b/src/packages/backend/nats/conf.ts index 5254dd6749..cb28729554 100644 --- a/src/packages/backend/nats/conf.ts +++ b/src/packages/backend/nats/conf.ts @@ -2,7 +2,7 @@ Configure nats-server, i.e., generate configuration files. -echo "await require('@cocalc/backend/nats/conf').configureNatsServer()" | node +node -e "require('@cocalc/backend/nats/conf').main()" */ @@ -16,6 +16,7 @@ import nsc from "./nsc"; import { executeCode } from "@cocalc/backend/execute-code"; import { startServer } from "./server"; import { kill } from "node:process"; +import { delay } from "awaiting"; const logger = getLogger("backend:nats:install"); @@ -63,9 +64,19 @@ ${await configureNsc()} ); const pid = startServer(); - // push initial operator/account/user configuration so its possible - // to configure other accounts - await nsc(["push", "-u", natsServerUrl]); + let d = 1000; + while (true) { + try { + // push initial operator/account/user configuration so its possible + // to configure other accounts + await nsc(["push", "-u", natsServerUrl]); + break; + } catch (err) { + console.log(err); + await delay(d); + d = Math.min(15000, d * 1.3); + } + } kill(pid); } @@ -102,3 +113,8 @@ export async function configureNsc() { const j = stdout.indexOf("\n", i + 1); return stdout.slice(0, j); } + +export async function main() { + await configureNatsServer(); + process.exit(0); +} diff --git a/src/packages/backend/nats/index.ts b/src/packages/backend/nats/index.ts index d8a8a669fa..ecc992433a 100644 --- a/src/packages/backend/nats/index.ts +++ b/src/packages/backend/nats/index.ts @@ -4,6 +4,8 @@ import { readFile } from "node:fs/promises"; import getLogger from "@cocalc/backend/logger"; import { connect, credsAuthenticator } from "nats"; export { getEnv } from "./env"; +import { delay } from "awaiting"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; const logger = getLogger("backend:nats"); @@ -19,18 +21,26 @@ export async function getCreds(): Promise { } } +let wait = 2000; let nc: Awaited> | null = null; -export async function getConnection() { +export const getConnection = reuseInFlight(async () => { logger.debug("connecting to nats"); - if (nc == null) { - const creds = await getCreds(); - nc = await connect({ - authenticator: credsAuthenticator(new TextEncoder().encode(creds)), - // bound on how long after network or server goes down until starts working again - pingInterval: 10000, - }); - logger.debug(`connected to ${nc.getServer()}`); + while (nc == null) { + try { + const creds = await getCreds(); + nc = await connect({ + authenticator: credsAuthenticator(new TextEncoder().encode(creds)), + // bound on how long after network or server goes down until starts working again + pingInterval: 10000, + }); + logger.debug(`connected to ${nc.getServer()}`); + } catch (err) { + logger.debug(`WARNING/ERROR: FAILED TO CONNECT TO nats-server: ${err}`); + logger.debug(`will retry in ${wait} ms`); + await delay(wait); + wait = Math.min(7500, 1.25 * wait); + } } return nc; -} +}); From 02b0af2427bcb5f2bee4b2e965d88e3c5e0e20d8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 04:01:29 +0000 Subject: [PATCH 093/281] nats: tiny bit more docs --- src/README.md | 8 +++++--- src/package.json | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/README.md b/src/README.md index 7747789b08..c26fe8e275 100644 --- a/src/README.md +++ b/src/README.md @@ -56,9 +56,11 @@ To install required dependencies, run hand, you prefer that development packages be installed globally, you can jump directly to the above `pip install` command outside the context of a virtual environment. -## Initial Build +## Build and Start -Launch the install and build **for doing development:** +Launch the install and build **for doing development.** + +If you export the PORT environment variable, that determines what port everything listens on. This determines subtle things about configuration, so do this once and for all in a consistent way. **Note**: If you installed `pnpm` locally (instead of globally), simply run `npm run` in place of `pnpm` to execute these commands via [NPM run scripts](https://docs.npmjs.com/cli/v10/using-npm/scripts). @@ -67,7 +69,7 @@ these commands via [NPM run scripts](https://docs.npmjs.com/cli/v10/using-npm/sc ~/cocalc/src$ pnpm build-dev ``` -This will do `pnpm install` for all packages, and also build the typescript/coffeescript, and anything else into a dist directory for each module. Once `pnpm build-dev` finishes successfully, you can start using CoCalc by starting the database, nats server and the backend hub in three terminals. \(Note that 'pnpm nats\-server' will download, install and configure NATS automatically.\) +This will do `pnpm install` for all packages, and also build the typescript/coffeescript, and anything else into a dist directory for each module. Once `pnpm build-dev` finishes successfully, you can start using CoCalc by starting the database, nats server and the backend hub in three terminals. \(Note that 'pnpm nats\-server' will download, install and configure NATS automatically.\) You can start the database, nats\-server and hub in any order. ```sh ~/cocalc/src$ pnpm database # in one terminal diff --git a/src/package.json b/src/package.json index 9b91651559..ee23cd2f42 100644 --- a/src/package.json +++ b/src/package.json @@ -19,6 +19,7 @@ "test": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r test", "prettier-all": "cd packages/", "nats-server": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/conf').main()\" && cd ${COCALC_ROOT:=$INIT_CWD}/data/nats && ./bin/nats-server -c server.conf", + "nats-server-verbose": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/conf').main()\" && cd ${COCALC_ROOT:=$INIT_CWD}/data/nats && ./bin/nats-server -DV -c server.conf", "nats": "echo; echo '# Put the following in your ~/.bashrc or use this subshell'; echo; echo \"export XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data\"; echo \"export XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data\"; echo \"export PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:\\$PATH\"; echo; echo; XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:$PATH bash" }, "repository": { From 937f5b8f0580f0a630900553b1f32059db36dc55 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 14:30:21 +0000 Subject: [PATCH 094/281] don't recomend nats env --- src/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/package.json b/src/package.json index ee23cd2f42..553a1cbf60 100644 --- a/src/package.json +++ b/src/package.json @@ -20,7 +20,7 @@ "prettier-all": "cd packages/", "nats-server": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/conf').main()\" && cd ${COCALC_ROOT:=$INIT_CWD}/data/nats && ./bin/nats-server -c server.conf", "nats-server-verbose": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/conf').main()\" && cd ${COCALC_ROOT:=$INIT_CWD}/data/nats && ./bin/nats-server -DV -c server.conf", - "nats": "echo; echo '# Put the following in your ~/.bashrc or use this subshell'; echo; echo \"export XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data\"; echo \"export XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data\"; echo \"export PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:\\$PATH\"; echo; echo; XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:$PATH bash" + "nats": "echo; echo '# Use CoCalc config of NATS (nats and nsc) via this subshell:'; echo; echo \"export XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data\"; echo \"export XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data\"; echo \"export PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:\\$PATH\"; echo; echo; XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:$PATH bash" }, "repository": { "type": "git", From 902802977705cf125e5af50d110f99f42bbeee3f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 17:06:31 +0000 Subject: [PATCH 095/281] nats database changefeeds -- rewrite how they work to address several subtle issues that would result in bugs and performance problems --- src/package.json | 2 +- src/packages/database/nats/index.ts | 177 +++++++++++++++--- src/packages/nats/sync/synctable-kv-atomic.ts | 2 +- src/packages/nats/sync/synctable-kv.ts | 2 +- src/packages/nats/sync/synctable-stream.ts | 2 + 5 files changed, 151 insertions(+), 34 deletions(-) diff --git a/src/package.json b/src/package.json index 553a1cbf60..723ba38b8a 100644 --- a/src/package.json +++ b/src/package.json @@ -20,7 +20,7 @@ "prettier-all": "cd packages/", "nats-server": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/conf').main()\" && cd ${COCALC_ROOT:=$INIT_CWD}/data/nats && ./bin/nats-server -c server.conf", "nats-server-verbose": "cd ${COCALC_ROOT:=$INIT_CWD}/packages/backend && node -e \"require('@cocalc/backend/nats/conf').main()\" && cd ${COCALC_ROOT:=$INIT_CWD}/data/nats && ./bin/nats-server -DV -c server.conf", - "nats": "echo; echo '# Use CoCalc config of NATS (nats and nsc) via this subshell:'; echo; echo \"export XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data\"; echo \"export XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data\"; echo \"export PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:\\$PATH\"; echo; echo; XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:$PATH bash" + "nats-cli": "echo; echo '# Use CoCalc config of NATS (nats and nsc) via this subshell:'; echo; echo \"export XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data\"; echo \"export XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data\"; echo \"export PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:\\$PATH\"; echo; echo; XDG_DATA_HOME=${COCALC_ROOT:=$INIT_CWD}/data XDG_CONFIG_HOME=${COCALC_ROOT:=$INIT_CWD}/data PATH=${COCALC_ROOT:=$INIT_CWD}/data/nats/bin:$PATH bash" }, "repository": { "type": "git", diff --git a/src/packages/database/nats/index.ts b/src/packages/database/nats/index.ts index cd0d3deb01..71801a2b15 100644 --- a/src/packages/database/nats/index.ts +++ b/src/packages/database/nats/index.ts @@ -27,9 +27,13 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { uuid } from "@cocalc/util/misc"; import { delay } from "awaiting"; import type { Subscription } from "nats"; +import { debounce } from "lodash"; const logger = getLogger("database:nats"); +const DEBOUNCE_SAVE_TO_JETSTREAM = 100; +const MAX_TIME_SAVE_TO_JETSTREAM = 30000; + const jc = JSONCodec(); let subscription: Subscription | null = null; @@ -48,6 +52,8 @@ export async function init() { export function terminate() { logger.debug("terminating"); subscription?.unsubscribe(); + // also, stop reporting data into the streams + cancelAllChangefeeds(); } async function handleRequest(mesg, nc) { @@ -89,11 +95,31 @@ function queryTable(query) { return Object.keys(query)[0]; } +const changefeedInterest: { [hash: string]: number } = {}; +const changefeedHashes: { [id: string]: string } = {}; + +function cancelChangefeed(id) { + logger.debug("cancelChangefeed", { id }); + const hash = changefeedHashes[id]; + if (!hash) { + // already canceled + return; + } + delete changefeedInterest[hash]; + delete changefeedHashes[id]; + db().user_query_cancel_changefeed({ id }); +} + +function cancelAllChangefeeds() { + logger.debug("cancelAllChangefeeds"); + for (const id in changefeedHashes) { + cancelChangefeed(id); + } +} + // This is tricky. We return the first result as a normal // async function, but then handle (and don't return) // the subsequent calls to cb generated by the changefeed. -const changefeedInterest: { [hash: string]: number } = {}; - const createChangefeed = reuseInFlight( async (opts, nc) => { const query = opts.query; @@ -106,6 +132,7 @@ const createChangefeed = reuseInFlight( } logger.debug("creating new changefeed for", queryTable(query)); const changes = uuid(); + changefeedHashes[changes] = hash; const env = { nc, jc, sha1 }; const synctable = createSyncTable({ query, @@ -115,42 +142,131 @@ const createChangefeed = reuseInFlight( atomic: true, }); await synctable.init(); + + /* + This code is complicated because it has to be. + + 1. The initial set could + take a long time and still be happening as we get updates + later. Thus we MUST use a work queue to ensure that every + update happens in the order it was received and also after + the initial state is set. + + 2. We keep a map in memory of the current value of all objects + so that in the case of an *update* we do not have to read + the last received value, which would take extra time and be + particularly hard given the queue issue in 1. + + 3. Saving to NATS could obviously fail intermittenly, e.g., if + NATS is down for some reason or there are network issues. + We retry with exponential backoff several times and + finally give up... TODO: user is not informed about this yet. + + */ + + const map: { [key: string]: object } = {}; + + const queue: { action: "insert" | "update" | "delete"; obj: object }[] = []; + + const synctableSetRows = async (rows) => { + if (rows.length == 0) { + return; + } + const v = rows.map(synctable.set); + // wait for confirmation that sets are done + let d = 2000; + let t = 0; + while ( + t < MAX_TIME_SAVE_TO_JETSTREAM && + changefeedHashes[changes] != null + ) { + const s = Date.now(); + try { + await Promise.all(v); + return; + } catch (err) { + logger.debug(`failed to save updates to NATS -- ${err}`); + await delay(d); + d = Math.min(10000, d * 1.3); + } + t += Date.now() - s; + } + logger.debug( + "WARNING: couldn't save to NATS after many attempts -- we cancel this whole changefeed", + ); + cancelChangefeed(changes); + }; + + const processQueue = debounce( + reuseInFlight(async () => { + const work = [...queue]; + // clear queue + queue.length = 0; + const rows: any[] = []; + for (const { action, obj } of work) { + if (action == "delete") { + // if we hit a delete, we have to handle everything up + // to this point, then do the delete. + await synctableSetRows(rows); + rows.length = 0; + await synctable.delete(obj); + } else { + rows.push(obj); + } + } + // handle anything left (will be everything if no deletes) + await synctableSetRows(rows); + }), + DEBOUNCE_SAVE_TO_JETSTREAM, + { leading: true, trailing: true }, + ); + + const handleFirst = ({ cb, err, rows }) => { + if (err || rows == null) { + cb(err ?? "missing result"); + return; + } + for (const obj of rows) { + map[synctable.getKey(obj)] = obj; + queue.push({ action: "insert", obj }); + } + processQueue(); + }; + + const handleUpdate = ({ action, new_val, old_val }) => { + // action = 'insert', 'update', 'delete', 'close' + // e.g., {"action":"insert","new_val":{"title":"testingxxxxx","project_id":"81e0c408-ac65-4114-bad5-5f4b6539bd0e"}} + const key = synctable.getKey(new_val ?? old_val); + if (action == "insert") { + map[key] = new_val; + queue.push({ action, obj: map[key] }); + } else if (action == "update") { + // update -- since atomic have to get the current value; + // this of course assumes there is one process writing to + // this part of the key value store (the atomic business). + map[key] = { ...map[key], ...new_val }; + queue.push({ action, obj: map[key] }); + } else if (action == "delete") { + delete map[key]; + queue.push({ action, obj: old_val }); + } else if (action == "close") { + cancelChangefeed(changes); + } + processQueue(); + }; + const f = (cb) => { let first = true; db().user_query({ ...opts, changes, - cb: async (err, result) => { + cb: (err, x) => { if (first) { first = false; - cb(err); - if (result != null) { - for (const x of result[synctable.table]) { - // logger.debug("changefeed init", x); - await synctable.set(x); - } - } + handleFirst({ cb, err, rows: x?.[synctable.table] }); return; } - // logger.debug("changefeed", result); - const { action, new_val, old_val } = result as any; - // action = 'insert', 'update', 'delete', 'close' - // e.g., {"action":"insert","new_val":{"title":"testingxxxxx","project_id":"81e0c408-ac65-4114-bad5-5f4b6539bd0e"}} - if (action == "insert") { - await synctable.set(new_val); - } else if (action == "update") { - // update -- since atomic have to get the current value; - // this of course assumes there is one process writing to - // this part of the key value store (the atomic business). - await synctable.set({ - ...(await synctable.get(new_val)), - ...new_val, - }); - } else if (action == "delete") { - await synctable.delete(old_val); - } else if (action == "close") { - delete changefeedInterest[hash]; - } + handleUpdate(x as any); }, }); }; @@ -171,8 +287,7 @@ const createChangefeed = reuseInFlight( "insufficient interest in the changefeed, so we stop it.", query, ); - db().user_query_cancel_changefeed({ id: changes }); - delete changefeedInterest[hash]; + cancelChangefeed(changes); } } }; diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index 32e2da01bc..a781cd619e 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -75,7 +75,7 @@ export class SyncTableKVAtomic extends EventEmitter { return this.sha1(this.primaryString(obj)); }; - private getKey = (obj): string => { + getKey = (obj): string => { return `${this.natsKeyPrefix}.${this.natObjectKey(obj)}`; }; diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 3fff8c2edd..8c46e347fc 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -346,7 +346,7 @@ export class SyncTableKV extends EventEmitter { return this.sha1(this.primaryString(this.fillInFromQuery(obj))); }; - private getKey = (obj, field?: string): string => { + getKey = (obj, field?: string): string => { const x = this.singleton ? this.natsKeyPrefix : `${this.natsKeyPrefix}.${this.natObjectKey(obj)}`; diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts index f2100007c2..bb44b221ce 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/nats/sync/synctable-stream.ts @@ -143,6 +143,8 @@ export class SyncTableStream extends EventEmitter { return toKey(this.primaryKeys.map((pk) => obj2[pk]))!; }; + getKey = this.primaryString; + private publish = (mesg) => { // console.log("publishing ", { subject: this.subject, mesg }); this.nc.publish(this.subject, this.jc.encode(mesg)); From 8412f2d093e4b435eaccbced7708846d50256fc6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 18:46:27 +0000 Subject: [PATCH 096/281] nats database synctable: implement a major optimization --- src/packages/database/nats/index.ts | 46 +++++++++++++++---- src/packages/nats/sync/synctable-kv-atomic.ts | 10 +++- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/packages/database/nats/index.ts b/src/packages/database/nats/index.ts index 71801a2b15..c269536061 100644 --- a/src/packages/database/nats/index.ts +++ b/src/packages/database/nats/index.ts @@ -146,6 +146,11 @@ const createChangefeed = reuseInFlight( /* This code is complicated because it has to be. + 0. Extra work to avoid ever setting a key in the nats kv if + we don't have to. Nat's doesn't do anything to avoid broadcasting + changes, so this is very valuable. For a supercluster it will + be a critical optimization. + 1. The initial set could take a long time and still be happening as we get updates later. Thus we MUST use a work queue to ensure that every @@ -164,9 +169,32 @@ const createChangefeed = reuseInFlight( */ - const map: { [key: string]: object } = {}; + // initalize map with exactly what is *currently* in the nats kv, + // so we can be sure to never set anything we don't need to set. + const map = await synctable.get(null, { natsKeys: true }); + const queue: { + action: "insert" | "update" | "delete"; + obj: object; + }[] = []; - const queue: { action: "insert" | "update" | "delete"; obj: object }[] = []; + const deleteMap = (key, obj) => { + if (map[key] !== undefined) { + delete map[key]; + queue.push({ action: "delete", obj }); + } + }; + + const setMap = (key, obj) => { + const cur = map[key]; + // always merge set + const value = { ...cur, ...obj }; + // json so dates compare as strings. Yes, we could make this faster, but this + // is entirely in the server. + if (JSON.stringify(cur) != JSON.stringify(value)) { + map[key] = value; + queue.push({ action: "update", obj: value }); + } + }; const synctableSetRows = async (rows) => { if (rows.length == 0) { @@ -227,8 +255,7 @@ const createChangefeed = reuseInFlight( return; } for (const obj of rows) { - map[synctable.getKey(obj)] = obj; - queue.push({ action: "insert", obj }); + setMap(synctable.getKey(obj), obj); } processQueue(); }; @@ -238,17 +265,14 @@ const createChangefeed = reuseInFlight( // e.g., {"action":"insert","new_val":{"title":"testingxxxxx","project_id":"81e0c408-ac65-4114-bad5-5f4b6539bd0e"}} const key = synctable.getKey(new_val ?? old_val); if (action == "insert") { - map[key] = new_val; - queue.push({ action, obj: map[key] }); + setMap(key, new_val); } else if (action == "update") { // update -- since atomic have to get the current value; // this of course assumes there is one process writing to // this part of the key value store (the atomic business). - map[key] = { ...map[key], ...new_val }; - queue.push({ action, obj: map[key] }); + setMap(key, new_val); } else if (action == "delete") { - delete map[key]; - queue.push({ action, obj: old_val }); + deleteMap(key, old_val); } else if (action == "close") { cancelChangefeed(changes); } @@ -295,6 +319,8 @@ const createChangefeed = reuseInFlight( watch(); return; } catch (err) { + // if anything goes wrong, make sure we don't think the changefeed is working. + cancelChangefeed(changes); throw err; } }, diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index a781cd619e..3adc576adc 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -93,12 +93,20 @@ export class SyncTableKVAtomic extends EventEmitter { return mesg?.sm?.data != null ? this.jc.decode(mesg.sm.data) : null; }; - get = async (obj?) => { + get = async (obj?, options: { natsKeys?: boolean } = {}) => { if (obj == null) { const raw = await getAllFromKv({ kv: this.kv, key: `${this.natsKeyPrefix}.>`, }); + if (options.natsKeys) { + // gets everything as a map with NATS keys but decoded values. + // This is used by the database changefeed stuff. + for (const key in raw) { + raw[key] = this.jc.decode(raw[key]); + } + return raw; + } const all: any = {}; for (const x of Object.values(raw)) { const value = this.jc.decode(x); From 090ebfca837cdb7f5f138366281c50331e772ba2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 20:34:13 +0000 Subject: [PATCH 097/281] sync editing -- commit more frequently --- src/packages/frontend/frame-editors/code-editor/const.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/packages/frontend/frame-editors/code-editor/const.ts b/src/packages/frontend/frame-editors/code-editor/const.ts index 3920ee6c53..cbf90da76d 100644 --- a/src/packages/frontend/frame-editors/code-editor/const.ts +++ b/src/packages/frontend/frame-editors/code-editor/const.ts @@ -7,7 +7,8 @@ // their changes are saved to time travel and broadcast to all other // users. -export const SAVE_DEBOUNCE_MS = 750; +// 50 words per minute is about 250ms between characters, so something slightly bigger than that. +export const SAVE_DEBOUNCE_MS = 500; // for testing sync issues manually, it is much easier with this large -- do not do this in production though! // export const SAVE_DEBOUNCE_MS = 3000; From 6d17cb8d286e4953a5240b3670059d57b2919ed7 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 22:03:09 +0000 Subject: [PATCH 098/281] nats: eliminate use of hub websocket for changefeeds... but - this breaks some things in crm editor still, and the old code is still there too --- src/packages/frontend/client/hub.ts | 2 +- src/packages/frontend/client/query.ts | 111 +++++++++--------- .../crm-editor/querydb/use-table.ts | 4 +- src/packages/nats/util.ts | 5 +- src/packages/sync/table/changefeed-nats.ts | 18 +-- src/packages/sync/table/changefeed.ts | 1 - src/packages/sync/table/synctable.ts | 1 - 7 files changed, 75 insertions(+), 67 deletions(-) diff --git a/src/packages/frontend/client/hub.ts b/src/packages/frontend/client/hub.ts index a4692eb223..a6e16ea077 100644 --- a/src/packages/frontend/client/hub.ts +++ b/src/packages/frontend/client/hub.ts @@ -113,7 +113,7 @@ export class HubClient { } public send(mesg: object): void { - // console.log("send to hub", mesg); + console.log("send to hub", mesg); const data = to_json_socket(mesg); this.mesg_data.sent_length += data.length; this.emit_mesg_data(); diff --git a/src/packages/frontend/client/query.ts b/src/packages/frontend/client/query.ts index 880477012a..c61934771a 100644 --- a/src/packages/frontend/client/query.ts +++ b/src/packages/frontend/client/query.ts @@ -3,90 +3,95 @@ * License: MS-RSL – see LICENSE.md for details */ -import * as message from "@cocalc/util/message"; import { is_array } from "@cocalc/util/misc"; import { validate_client_query } from "@cocalc/util/schema-validate"; import { CB } from "@cocalc/util/types/database"; +import { NatsChangefeed } from "@cocalc/sync/table/changefeed-nats"; +import { uuid } from "@cocalc/util/misc"; declare const $: any; // jQuery export class QueryClient { private client: any; + private changefeeds: { [id: string]: NatsChangefeed } = {}; constructor(client: any) { this.client = client; } - private async call(message: object, timeout: number): Promise { - return await this.client.async_call({ - message, - timeout, - allow_post: false, // since that would happen via this.post_query - }); - } - // This works like a normal async function when // opts.cb is NOT specified. When opts.cb is specified, - // it works like a cb and returns nothing. For changefeeds + // it works like a cb and returns nothing. For changefeeds // you MUST specify opts.cb, but can always optionally do so. public async query(opts: { query: object; - changes?: boolean; options?: object[]; // if given must be an array of objects, e.g., [{limit:5}] - standby?: boolean; // if true and use HTTP post, then will use standby server (so must be read only) - timeout?: number; // default: 30 - no_post?: boolean; // DEPRECATED -- didn't turn out to be worth it. - ignore_response?: boolean; // if true, be slightly efficient by not waiting for any response or - // error (just assume it worked; don't care about response) - cb?: CB; // used for changefeed outputs if changes is true + changes?: boolean; + cb?: CB; // support old cb interface }): Promise { + // Deprecation warnings: + for (const field of ["standby", "timeout", "no_post", "ignore_response"]) { + if (opts[field] != null) { + console.trace(`WARNING: passing '${field}' to query is deprecated`); + } + } if (opts.options != null && !is_array(opts.options)) { // should never happen... throw Error("options must be an array"); } - if (opts.changes && opts.cb == null) { - throw Error("for changefeed, must specify opts.cb"); - } - - const err = validate_client_query(opts.query, this.client.account_id); - if (err) { - throw Error(err); - } - const mesg = message.query({ - query: opts.query, - options: opts.options, - changes: opts.changes, - multi_response: !!opts.changes, - }); - if (opts.timeout == null) { - opts.timeout = 30; - } - if (mesg.multi_response) { - if (opts.cb == null) { - throw Error("changefeed requires cb callback"); + if (opts.changes) { + const { cb } = opts; + if (cb == null) { + throw Error("for changefeed, must specify opts.cb"); + } + let changefeed; + try { + changefeed = new NatsChangefeed({ + client: this.client, + query: opts.query, // todo: regarding options + }); + // id for canceling this changefeed + const id = uuid(); + const rows = await changefeed.connect(); + const query = { [Object.keys(opts.query)[0]]: rows }; + this.changefeeds[id] = changefeed; + cb(undefined, { query, id }); + changefeed.on("update", (change) => { + cb(undefined, change); + }); + } catch (err) { + cb(`${err}`); + return; } - this.client.call({ - allow_post: false, - message: mesg, - error_event: true, - timeout: opts.timeout, - cb: opts.cb, - }); } else { - if (opts.cb != null) { - try { - const result = await this.call(mesg, opts.timeout); - opts.cb(undefined, result); - } catch (err) { - opts.cb(typeof err == "string" ? err : err.message ?? err); + try { + const err = validate_client_query(opts.query, this.client.account_id); + if (err) { + throw Error(err); + } + const result = await this.client.nats_client.hub.db.userQuery({ + query: opts.query, + options: opts.options, + }); + if (opts.cb == null) { + return result; + } else { + opts.cb(undefined, { query: result }); + } + } catch (err) { + if (opts.cb == null) { + throw err; + } else { + opts.cb(err); } - } else { - return await this.call(mesg, opts.timeout); } } } + // cancel a changefeed created above. This is ONLY used + // right now by the CRM code. public async cancel(id: string): Promise { - await this.call(message.query_cancel({ id }), 30); + this.changefeeds[id]?.close(); + delete this.changefeeds[id]; } } diff --git a/src/packages/frontend/frame-editors/crm-editor/querydb/use-table.ts b/src/packages/frontend/frame-editors/crm-editor/querydb/use-table.ts index 3f034da535..82559e0d77 100644 --- a/src/packages/frontend/frame-editors/crm-editor/querydb/use-table.ts +++ b/src/packages/frontend/frame-editors/crm-editor/querydb/use-table.ts @@ -136,7 +136,7 @@ export function useTable({ const x = { id: "" }; const q = getQuery(query, hiddenFields, search); const options = ([{ limit: limit ?? DEFAULT_LIMIT }] as any[]).concat( - sortOptions(sortFields) + sortOptions(sortFields), ); setLoading(true); webapp_client.query_client.query({ @@ -187,7 +187,7 @@ export function useTable({ }; }, }, - [disconnectCounter, sortFields, hiddenFields, limit, search] + [disconnectCounter, sortFields, hiddenFields, limit, search], ); const refresh = incDisconnectCounter; diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index 10a6108c85..1cdd46c62d 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -1,4 +1,4 @@ -// Get the number of keys in a nats kv store, matching a given subject: +// Get the number of NON-deleted keys in a nats kv store, matching a given subject: export async function numKeys(kv, x: string | string[] = ">"): Promise { let num = 0; for await (const _ of await kv.keys(x)) { @@ -31,10 +31,11 @@ export async function getAllFromKv({ if (total == 0) { return all; } - const watch = await kv.watch({ key }); + const watch = await kv.watch({ key, ignoreDeletes: true }); let id: any = 0; for await (const { key, value } of watch) { all[key] = value; + count += 1; if (id) { diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index 61616405d4..891ce3b108 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -13,8 +13,9 @@ export class NatsChangefeed extends EventEmitter { private options; private state: State = "disconnected"; private natsSynctable?; + private watch?; - constructor({ client, query, options }: { client; query; options }) { + constructor({ client, query, options }: { client; query; options? }) { super(); this.client = client; this.query = query; @@ -27,11 +28,15 @@ export class NatsChangefeed extends EventEmitter { connect = async () => { this.natsSynctable = await this.client.nats_client.changefeed(this.query); this.interest(); - this.watch(); + this.startWatch(); return Object.values(await this.natsSynctable.get()); }; close = (): void => { + if (this.watch != null) { + this.watch.stop(); + delete this.watch; + } this.state = "closed"; this.emit("close"); }; @@ -48,14 +53,13 @@ export class NatsChangefeed extends EventEmitter { await delay(30000); } }; - private watch = async () => { + + private startWatch = async () => { if (this.natsSynctable == null) { return; } - for await (const new_val of await this.natsSynctable.watch()) { - if (this.state == "closed") { - return; - } + this.watch = await this.natsSynctable.watch(); + for await (const new_val of this.watch) { this.emit("update", { action: "update", new_val }); } }; diff --git a/src/packages/sync/table/changefeed.ts b/src/packages/sync/table/changefeed.ts index 4b3b1470ed..18c1eccd06 100644 --- a/src/packages/sync/table/changefeed.ts +++ b/src/packages/sync/table/changefeed.ts @@ -107,7 +107,6 @@ export class Changefeed extends EventEmitter { this.do_query({ query: this.query, changes: true, - timeout: 30, options: this.options, cb: (err, resp) => { if (first_time) { diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index 77955e9918..baa87a6490 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -1050,7 +1050,6 @@ export class SyncTable extends EventEmitter { await callback2(this.client.query, { query, options: [{ set: true }], // force it to be a set query - timeout: 120, // give it some time (especially if it is long) }); this.last_save = value; // success -- don't have to save this stuff anymore... } catch (err) { From 6b43eb7172aea52883461daa7030d2c78b53231d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 23:36:30 +0000 Subject: [PATCH 099/281] nats query client: fix return format so now crm works fine :-) --- src/packages/frontend/client/query.ts | 6 ++--- .../crm-editor/fields/select.tsx | 10 +++---- .../crm-editor/views/create-new-record.ts | 27 +++++++++---------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/packages/frontend/client/query.ts b/src/packages/frontend/client/query.ts index c61934771a..f24adff13e 100644 --- a/src/packages/frontend/client/query.ts +++ b/src/packages/frontend/client/query.ts @@ -69,14 +69,14 @@ export class QueryClient { if (err) { throw Error(err); } - const result = await this.client.nats_client.hub.db.userQuery({ + const query = await this.client.nats_client.hub.db.userQuery({ query: opts.query, options: opts.options, }); if (opts.cb == null) { - return result; + return { query }; } else { - opts.cb(undefined, { query: result }); + opts.cb(undefined, { query }); } } catch (err) { if (opts.cb == null) { diff --git a/src/packages/frontend/frame-editors/crm-editor/fields/select.tsx b/src/packages/frontend/frame-editors/crm-editor/fields/select.tsx index dcb7657102..f66de81de2 100644 --- a/src/packages/frontend/frame-editors/crm-editor/fields/select.tsx +++ b/src/packages/frontend/frame-editors/crm-editor/fields/select.tsx @@ -1,6 +1,6 @@ import { ReactNode, useEffect, useMemo, useState } from "react"; import { render, sorter, ANY } from "./register"; -import { Progress, Select, Tag } from "antd"; +import { Progress, Select, Tag, Space } from "antd"; import { capitalize, cmp } from "@cocalc/util/misc"; import { useEditableContext } from "./context"; import LRU from "lru-cache"; @@ -13,7 +13,7 @@ function StatusDisplay({ value, color, n }) { function PriorityDisplay({ value, color, n, len }) { if (n == -1) return null; return ( -
+
{capitalize(value)}
-
+ ); } @@ -77,7 +77,7 @@ render( } const { options, valueDisplay, valueToNumber } = useMemo( () => parse(spec), - [spec] + [spec], ); const { counter, save, error } = useEditableContext(field); @@ -119,7 +119,7 @@ render( {error} ); - } + }, ); sorter({ type: "select", options: ANY, colors: ANY, priority: ANY }, (spec) => { diff --git a/src/packages/frontend/frame-editors/crm-editor/views/create-new-record.ts b/src/packages/frontend/frame-editors/crm-editor/views/create-new-record.ts index 488c0584a3..c0a705084b 100644 --- a/src/packages/frontend/frame-editors/crm-editor/views/create-new-record.ts +++ b/src/packages/frontend/frame-editors/crm-editor/views/create-new-record.ts @@ -65,19 +65,18 @@ async function create(dbtable, obj): Promise { // is likely to be ours. // TODO: this is fine until we do this properly since probably it's // just one admin manually using this. - const recent = ( - await webapp_client.async_query({ - query: { - [dbtable]: [ - { - id: null, - created: { ">=": { relative_time: -15, unit: "seconds" } }, - }, - ], - }, - options: [{ order_by: "-created" }], - }) - ).query[dbtable]; + const result = await webapp_client.async_query({ + query: { + [dbtable]: [ + { + id: null, + created: { ">=": { relative_time: -15, unit: "seconds" } }, + }, + ], + }, + options: [{ order_by: "-created" }], + }); + const recent = result.query[dbtable]; return recent[0].id; } @@ -85,7 +84,7 @@ async function create(dbtable, obj): Promise { function fillInAtomicSearch( x, dbtable, - { field, operator, value }: AtomicSearch + { field, operator, value }: AtomicSearch, ) { if (!field || !operator || value === undefined) { // only partially input so not being used. (TODO?) From 4653cc2343cc4d0ee6db9f4c587ba90f704516d2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 2 Feb 2025 23:58:03 +0000 Subject: [PATCH 100/281] nats: work in progress refactoring touch, touch_project --- src/packages/nats/hub-api/db.ts | 8 ++++++++ src/packages/server/nats/api/db.ts | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/packages/nats/hub-api/db.ts b/src/packages/nats/hub-api/db.ts index 0647b801c3..74afde652c 100644 --- a/src/packages/nats/hub-api/db.ts +++ b/src/packages/nats/hub-api/db.ts @@ -2,6 +2,7 @@ import { authFirst } from "./util"; export const db = { userQuery: authFirst, + touch: authFirst, }; export interface DB { @@ -11,4 +12,11 @@ export interface DB { query: any; options?: any[]; }) => Promise; + + touch: (opts: { + account_id: string; + project_id?: string; + path?: string; + action?: string; + }) => Promise; } diff --git a/src/packages/server/nats/api/db.ts b/src/packages/server/nats/api/db.ts index 6863fd9e01..58327994f0 100644 --- a/src/packages/server/nats/api/db.ts +++ b/src/packages/server/nats/api/db.ts @@ -1,3 +1,29 @@ +import { db } from "@cocalc/database"; +import isCollaborator from "@cocalc/server/projects/is-collaborator"; import userQuery from "@cocalc/database/user-query"; export { userQuery }; + +export async function touch({ + account_id, + project_id, + path, + action = "edit", +}: { + account_id: string; + project_id?: string; + path?: string; + action?: string; +}): Promise { + if (!project_id) { + await db().touch({ account_id, action }); + return; + } + if (!(await isCollaborator({ account_id, project_id }))) { + throw Error("user must be collaborator on project"); + } + const D = db(); + D.ensure_connection_to_project(project_id); + await D.touch({ account_id, project_id, path, action }); + // TODO: we also connect still (this will of course go away very soon!!) +} From 3dd81a260a4cf34557721ddecc734fccc181cc13 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 3 Feb 2025 01:16:12 +0000 Subject: [PATCH 101/281] nats: eliminate old touch codepath -- use new api --- src/packages/database/postgres/types.ts | 8 +- src/packages/frontend/billing/actions.ts | 3 + src/packages/frontend/client/project.ts | 2 +- src/packages/frontend/client/stripe.ts | 2 + src/packages/hub/client.coffee | 116 ----------------------- src/packages/nats/hub-api/db.ts | 2 +- src/packages/server/nats/api/db.ts | 14 ++- src/packages/server/nats/api/index.ts | 1 + src/packages/util/message.d.ts | 1 - src/packages/util/message.js | 18 ---- 10 files changed, 24 insertions(+), 143 deletions(-) diff --git a/src/packages/database/postgres/types.ts b/src/packages/database/postgres/types.ts index b5c4c622e0..37bfcb2ebd 100644 --- a/src/packages/database/postgres/types.ts +++ b/src/packages/database/postgres/types.ts @@ -346,7 +346,13 @@ export interface PostgreSQL extends EventEmitter { set_project_status(opts: { project_id: string; status: ProjectStatus }): void; - touch(opts: { project_id: string; account_id: string; cb: CB }); + touch(opts: { + project_id?: string; + account_id: string; + action?: string; + path?: string; + cb: CB; + }); get_project_extra_env(opts: { project_id: string; cb: CB }): void; diff --git a/src/packages/frontend/billing/actions.ts b/src/packages/frontend/billing/actions.ts index dc8cd42b44..2751c309e8 100644 --- a/src/packages/frontend/billing/actions.ts +++ b/src/packages/frontend/billing/actions.ts @@ -3,6 +3,9 @@ * License: MS-RSL – see LICENSE.md for details */ +// COMPLETEY DEPRECATED -- DELETE THIS ? + + /* Billing actions. diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index f0ad9f5021..b3da21bfd9 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -457,7 +457,7 @@ export class ProjectClient { } this.touch_throttle[project_id] = Date.now(); try { - await this.call(message.touch_project({ project_id })); + await this.client.nats_client.hub.db.touch({ project_id }); } catch (err) { // silently ignore; this happens, e.g., if you touch too frequently, // and shouldn't be fatal and break other things. diff --git a/src/packages/frontend/client/stripe.ts b/src/packages/frontend/client/stripe.ts index 5f44c24500..6462047088 100644 --- a/src/packages/frontend/client/stripe.ts +++ b/src/packages/frontend/client/stripe.ts @@ -3,6 +3,8 @@ * License: MS-RSL – see LICENSE.md for details */ +// COMPLETEY DEPRECATED -- DELETE THIS ? + /* stripe payments api via backend hub */ diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee index e33de54f4e..0ee53cced3 100644 --- a/src/packages/hub/client.coffee +++ b/src/packages/hub/client.coffee @@ -1570,91 +1570,6 @@ class exports.Client extends EventEmitter @push_to_client(message.success(id:mesg.id)) ) - ### - Data Query - ### - mesg_query: (mesg) => - dbg = @dbg("user_query") - query = mesg.query - if not query? - @error_to_client(id:mesg.id, error:"malformed query") - return - # CRITICAL: don't enable this except for serious debugging, since it can result in HUGE output - #dbg("account_id=#{@account_id} makes query='#{misc.to_json(query)}'") - first = true - if mesg.changes - @_query_changefeeds ?= {} - @_query_changefeeds[mesg.id] = true - mesg_id = mesg.id - @database.user_query - client_id : @id - account_id : @account_id - query : query - options : mesg.options - changes : if mesg.changes then mesg_id - cb : (err, result) => - if @closed # connection closed, so nothing further to do with this - return - if result?.action == 'close' - err = 'close' - if err - dbg("user_query(query='#{misc.to_json(query)}') error:", err) - if @_query_changefeeds?[mesg_id] - delete @_query_changefeeds[mesg_id] - @error_to_client(id:mesg_id, error:"#{err}") # Ensure err like Error('foo') can be JSON'd - if mesg.changes and not first and @_query_changefeeds?[mesg_id]? - dbg("changefeed got messed up, so cancel it:") - @database.user_query_cancel_changefeed(id : mesg_id) - else - if mesg.changes and not first - resp = result - resp.id = mesg_id - resp.multi_response = true - else - first = false - resp = mesg - resp.query = result - @push_to_client(resp) - - query_cancel_all_changefeeds: (cb) => - if not @_query_changefeeds? - cb?(); return - cnt = misc.len(@_query_changefeeds) - if cnt == 0 - cb?(); return - dbg = @dbg("query_cancel_all_changefeeds") - v = @_query_changefeeds - dbg("cancel #{cnt} changefeeds") - delete @_query_changefeeds - f = (id, cb) => - dbg("cancel id=#{id}") - @database.user_query_cancel_changefeed - id : id - cb : (err) => - if err - dbg("FEED: warning #{id} -- error canceling a changefeed #{misc.to_json(err)}") - else - dbg("FEED: canceled changefeed -- #{id}") - cb() - async.map(misc.keys(v), f, (err) => cb?(err)) - - mesg_query_cancel: (mesg) => - if not @_query_changefeeds?[mesg.id]? - # no such changefeed - @success_to_client(id:mesg.id) - else - # actualy cancel it. - if @_query_changefeeds? - delete @_query_changefeeds[mesg.id] - @database.user_query_cancel_changefeed - id : mesg.id - cb : (err, resp) => - if err - @error_to_client(id:mesg.id, error:err) - else - mesg.resp = resp - @push_to_client(mesg) - ### Support Tickets → Zendesk @@ -1867,37 +1782,6 @@ class exports.Client extends EventEmitter local_hub_connection.disconnect_from_project(mesg.project_id) @push_to_client(message.success(id:mesg.id)) - mesg_touch_project: (mesg) => - dbg = @dbg('mesg_touch_project') - async.series([ - (cb) => - dbg("checking conditions") - @_check_project_access(mesg.project_id, cb) - (cb) => - # IMPORTANT: do this ensure_connection_to_project *first*, since - # it is critical always ensure this, and the @touch below gives - # an error if done more than once per 45s, whereas we may want - # to check much more frequently that we have a TCP connection - # to the project. - f = @database.ensure_connection_to_project - if f? - dbg("also create socket connection (so project can query db, etc.)") - # We do NOT block on this -- it can take a while. - f(mesg.project_id) - cb() - (cb) => - @touch - project_id : mesg.project_id - action : 'touch' - cb : cb - ], (err) => - if err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"unable to touch project #{mesg.project_id} -- #{err}") - else - @push_to_client(message.success(id:mesg.id)) - ) - mesg_get_syncdoc_history: (mesg) => dbg = @dbg('mesg_syncdoc_history') try diff --git a/src/packages/nats/hub-api/db.ts b/src/packages/nats/hub-api/db.ts index 74afde652c..cfb39d5086 100644 --- a/src/packages/nats/hub-api/db.ts +++ b/src/packages/nats/hub-api/db.ts @@ -14,7 +14,7 @@ export interface DB { }) => Promise; touch: (opts: { - account_id: string; + account_id?: string; project_id?: string; path?: string; action?: string; diff --git a/src/packages/server/nats/api/db.ts b/src/packages/server/nats/api/db.ts index 58327994f0..876e832bad 100644 --- a/src/packages/server/nats/api/db.ts +++ b/src/packages/server/nats/api/db.ts @@ -1,6 +1,7 @@ import { db } from "@cocalc/database"; import isCollaborator from "@cocalc/server/projects/is-collaborator"; import userQuery from "@cocalc/database/user-query"; +import { callback2 } from "@cocalc/util/async-utils"; export { userQuery }; @@ -10,20 +11,23 @@ export async function touch({ path, action = "edit", }: { - account_id: string; + account_id?: string; project_id?: string; path?: string; action?: string; }): Promise { + const D = db(); + if (!account_id) { + throw Error("account_id must be set"); + } if (!project_id) { - await db().touch({ account_id, action }); + await callback2(D.touch, { account_id, action }); return; } if (!(await isCollaborator({ account_id, project_id }))) { throw Error("user must be collaborator on project"); } - const D = db(); - D.ensure_connection_to_project(project_id); - await D.touch({ account_id, project_id, path, action }); // TODO: we also connect still (this will of course go away very soon!!) + D.ensure_connection_to_project?.(project_id); + await callback2(D.touch, { account_id, project_id, path, action }); } diff --git a/src/packages/server/nats/api/index.ts b/src/packages/server/nats/api/index.ts index 20c19f100f..1e468fb47d 100644 --- a/src/packages/server/nats/api/index.ts +++ b/src/packages/server/nats/api/index.ts @@ -64,6 +64,7 @@ export async function initAPI() { // TODO: should be part of handleApiRequest below, but done differently because // one case halts this loop const { service } = request.args[0] ?? {}; + logger.debug(`Terminate service '${service}'`); if (service == "database") { terminateDatabase(); mesg.respond(jc.encode({ status: "terminating", service })); diff --git a/src/packages/util/message.d.ts b/src/packages/util/message.d.ts index c3fff78cef..6b2470946c 100644 --- a/src/packages/util/message.d.ts +++ b/src/packages/util/message.d.ts @@ -102,7 +102,6 @@ export const user_auth_token: any; export const metrics: any; export const start_metrics: any; export const get_available_upgrades: any; -export const touch_project: any; export const disconnect_from_project: any; export const available_upgrades: any; export const remove_all_upgrades: any; diff --git a/src/packages/util/message.js b/src/packages/util/message.js index 9e459601a6..1d17d1a5c3 100644 --- a/src/packages/util/message.js +++ b/src/packages/util/message.js @@ -2701,24 +2701,6 @@ Example: }), ); -// client --> hub -API( - message2({ - event: "touch_project", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - project_id: { - init: required, - desc: "id of project to touch", - }, - }, - desc: "Mark this project as being actively used by the user sending this message. This keeps the project from idle timing out, among other things.", - }), -); - // client --> hub API( message2({ From e97c18bb8beabe0143e4a4d2c97d5aacb4e8e995 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 3 Feb 2025 03:18:15 +0000 Subject: [PATCH 102/281] user_tracking: switch to nats api; at admin setting so it is disabled at default and easy to disable/enable --- src/packages/database/settings/customize.ts | 2 ++ .../frontend/admin/site-settings/index.tsx | 16 ++++------ src/packages/frontend/client/tracking.ts | 29 ++++++++++++------- src/packages/frontend/customize.tsx | 4 ++- src/packages/hub/client.coffee | 11 ------- src/packages/hub/webapp-configuration.ts | 1 - src/packages/nats/hub-api/system.ts | 12 ++++++++ src/packages/server/nats/api/system.ts | 14 +++++++++ src/packages/util/db-schema/site-defaults.ts | 9 +++++- 9 files changed, 63 insertions(+), 35 deletions(-) diff --git a/src/packages/database/settings/customize.ts b/src/packages/database/settings/customize.ts index 549431f49b..f6d943f03f 100644 --- a/src/packages/database/settings/customize.ts +++ b/src/packages/database/settings/customize.ts @@ -126,6 +126,8 @@ export default async function getCustomize( strategies, verifyEmailAddresses: settings.verify_emails && settings.email_enabled, + + userTracking: settings.user_tracking, }; } return fields ? copy_with(cachedCustomize, fields) : cachedCustomize; diff --git a/src/packages/frontend/admin/site-settings/index.tsx b/src/packages/frontend/admin/site-settings/index.tsx index 4520698f33..babe4d562e 100644 --- a/src/packages/frontend/admin/site-settings/index.tsx +++ b/src/packages/frontend/admin/site-settings/index.tsx @@ -32,6 +32,7 @@ import { toCustomOpenAIModel, toOllamaModel, } from "@cocalc/util/db-schema/llm-utils"; +import ShowError from "@cocalc/frontend/components/error"; const { CheckableTag } = AntdTag; @@ -461,16 +462,11 @@ export default function SiteSettings({ close }) { }} > - {error && ( - setError("")} - style={{ margin: "30px auto", maxWidth: "800px" }} - /> - )} + diff --git a/src/packages/frontend/client/tracking.ts b/src/packages/frontend/client/tracking.ts index b3ce5bcda6..05465d25d2 100644 --- a/src/packages/frontend/client/tracking.ts +++ b/src/packages/frontend/client/tracking.ts @@ -5,10 +5,12 @@ import { WebappClient } from "./client"; import * as message from "@cocalc/util/message"; +import { redux } from "@cocalc/frontend/app-framework"; export class TrackingClient { private client: WebappClient; private log_error_cache: { [error: string]: number } = {}; + private userTrackingEnabled?: string; constructor(client: WebappClient) { this.client = client; @@ -17,17 +19,22 @@ export class TrackingClient { // Send metrics to the hub this client is connected to. // There is no confirmation or response that this succeeded, // which is fine, since dropping some metrics is fine. - public send_metrics(metrics: object): void { + send_metrics = (metrics: object): void => { this.client.hub_client.send(message.metrics({ metrics })); - } + }; - public async user_tracking(evt: string, value: object): Promise { - await this.client.async_call({ - message: message.user_tracking({ evt, value }), - }); - } + user_tracking = async (event: string, value: object): Promise => { + if (this.userTrackingEnabled == null) { + this.userTrackingEnabled = redux + .getStore("customize") + ?.get("user_tracking"); + } + if (this.userTrackingEnabled == "yes") { + await this.client.nats_client.hub.system.userTracking({ event, value }); + } + }; - public log_error(error: any): void { + log_error = (error: any): void => { if (typeof error != "string") { error = JSON.stringify(error); } @@ -39,9 +46,9 @@ export class TrackingClient { this.client.call({ message: message.log_client_error({ error }), }); - } + }; - public async webapp_error(opts: object): Promise { + webapp_error = async (opts: object): Promise => { await this.client.async_call({ message: message.webapp_error(opts) }); - } + }; } diff --git a/src/packages/frontend/customize.tsx b/src/packages/frontend/customize.tsx index 63ea3c4eae..11a21e8e4d 100644 --- a/src/packages/frontend/customize.tsx +++ b/src/packages/frontend/customize.tsx @@ -181,6 +181,8 @@ export interface CustomizeState { insecure_test_mode?: boolean; i18n?: List; + + user_tracking?: string; } export class CustomizeStore extends Store { @@ -332,7 +334,7 @@ function process_customize(obj) { for (const k in site_settings_conf) { const v = site_settings_conf[k]; obj[k] = - obj[k] != null ? obj[k] : v.to_val?.(v.default, obj_orig) ?? v.default; + obj[k] != null ? obj[k] : (v.to_val?.(v.default, obj_orig) ?? v.default); } // the llm markup special case obj.llm_markup = obj_orig._llm_markup ?? 30; diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee index 0ee53cced3..8a1cf2348c 100644 --- a/src/packages/hub/client.coffee +++ b/src/packages/hub/client.coffee @@ -1796,17 +1796,6 @@ class exports.Client extends EventEmitter dbg("failed -- #{err}") @error_to_client(id:mesg.id, error:"unable to get syncdoc history for string_id #{mesg.string_id} -- #{err}") - mesg_user_tracking: (mesg) => - dbg = @dbg("mesg_user_tracking") - try - if not @account_id - throw Error("you must be signed in to record a tracking event") - await record_user_tracking(@database, @account_id, mesg.evt, mesg.value) - @push_to_client(message.success(id:mesg.id)) - catch err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"unable to record user_tracking event #{mesg.evt} -- #{err}") - mesg_admin_reset_password: (mesg) => dbg = @dbg("mesg_reset_password") dbg(mesg.email_address) diff --git a/src/packages/hub/webapp-configuration.ts b/src/packages/hub/webapp-configuration.ts index 787038b7f9..3b5a636b33 100644 --- a/src/packages/hub/webapp-configuration.ts +++ b/src/packages/hub/webapp-configuration.ts @@ -13,7 +13,6 @@ import { delay } from "awaiting"; import debug from "debug"; import { isEmpty } from "lodash"; import LRU from "lru-cache"; - import type { PostgreSQL } from "@cocalc/database/postgres/types"; import { get_passport_manager, PassportManager } from "@cocalc/server/hub/auth"; import { getSoftwareEnvironments } from "@cocalc/server/software-envs"; diff --git a/src/packages/nats/hub-api/system.ts b/src/packages/nats/hub-api/system.ts index b78aa7b2b8..1dcf4f5698 100644 --- a/src/packages/nats/hub-api/system.ts +++ b/src/packages/nats/hub-api/system.ts @@ -6,11 +6,23 @@ export const system = { ping: noAuth, addProjectPermission: authFirst, terminate: authFirst, + userTracking: authFirst, }; export interface System { + // get all or specific customize data getCustomize: (fields?: string[]) => Promise; + // ping server and get back the current time ping: () => { now: number }; + // request to have NATS permissions to project subjects. addProjectPermission: (opts: { project_id: string }) => Promise; + // terminate a service: + // - only admin can do this. + // - useful for development terminate: (service: "database" | "api") => Promise; + userTracking: (opts: { + event: string; + value: object; + account_id?: string; + }) => Promise; } diff --git a/src/packages/server/nats/api/system.ts b/src/packages/server/nats/api/system.ts index 5e0fd7337d..3f89af0fed 100644 --- a/src/packages/server/nats/api/system.ts +++ b/src/packages/server/nats/api/system.ts @@ -2,9 +2,23 @@ import getCustomize from "@cocalc/database/settings/customize"; export { getCustomize }; import { addProjectPermission } from "@cocalc/server/nats/auth"; export { addProjectPermission }; +import { record_user_tracking } from "@cocalc/database/postgres/user-tracking"; +import { db } from "@cocalc/database"; export function ping() { return { now: Date.now() }; } export async function terminate() {} + +export async function userTracking({ + event, + value, + account_id, +}: { + event: string; + value: object; + account_id?: string; +}): Promise { + await record_user_tracking(db(), account_id!, event, value); +} diff --git a/src/packages/util/db-schema/site-defaults.ts b/src/packages/util/db-schema/site-defaults.ts index efa43cdbb4..8ff9fff7c0 100644 --- a/src/packages/util/db-schema/site-defaults.ts +++ b/src/packages/util/db-schema/site-defaults.ts @@ -119,7 +119,8 @@ export type SiteSettingsKeys = | "compute_servers_hyperstack_enabled" | "cloud_filesystems_enabled" | "insecure_test_mode" - | "samesite_remember_me"; + | "samesite_remember_me" + | "user_tracking"; //| "compute_servers_lambda-cloud_enabled" @@ -972,4 +973,10 @@ export const site_settings_conf: SiteSettings = { to_val: (x) => `${x}`, tags: ["Security"], }, + user_tracking: { + name: "User Tracking", + desc: "If enabled, then information about what users do in the frontend browser gets temporarily recorded in the user_tracking table of the database.", + default: "no", + valid: only_booleans, + }, } as const; From 092bfc5bcc426421b65693d69cdc7d5d3d78dc4c Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 3 Feb 2025 03:40:42 +0000 Subject: [PATCH 103/281] remove web browser prometheus reporting - awkward to do with no persistent connection to a single hub (which we will not have with nats) - most of the data is not relevant anymore - most code ancient coffeescript - package itself was several versions old - i never looked at this data even once --- src/packages/database/settings/customize.ts | 2 - src/packages/frontend/app/monitor-pings.ts | 18 -- src/packages/frontend/client/client.ts | 6 - src/packages/frontend/client/console.ts | 1 - src/packages/frontend/client/hub.ts | 3 - src/packages/frontend/client/tracking.ts | 7 - src/packages/frontend/last.ts | 22 --- src/packages/frontend/package.json | 1 - .../frontend/project/directory-listing.ts | 32 ---- src/packages/frontend/prom-client.ts | 123 ------------ src/packages/hub/client.coffee | 58 ------ src/packages/hub/metrics-recorder.coffee | 181 ------------------ src/packages/pnpm-lock.yaml | 3 - src/packages/server/nats/api/system.ts | 1 + src/packages/util/message.d.ts | 2 - src/packages/util/message.js | 20 -- 16 files changed, 1 insertion(+), 479 deletions(-) delete mode 100644 src/packages/frontend/prom-client.ts delete mode 100644 src/packages/hub/metrics-recorder.coffee diff --git a/src/packages/database/settings/customize.ts b/src/packages/database/settings/customize.ts index f6d943f03f..549431f49b 100644 --- a/src/packages/database/settings/customize.ts +++ b/src/packages/database/settings/customize.ts @@ -126,8 +126,6 @@ export default async function getCustomize( strategies, verifyEmailAddresses: settings.verify_emails && settings.email_enabled, - - userTracking: settings.user_tracking, }; } return fields ? copy_with(cachedCustomize, fields) : cachedCustomize; diff --git a/src/packages/frontend/app/monitor-pings.ts b/src/packages/frontend/app/monitor-pings.ts index 5147545102..6140dad46c 100644 --- a/src/packages/frontend/app/monitor-pings.ts +++ b/src/packages/frontend/app/monitor-pings.ts @@ -8,21 +8,8 @@ import { redux } from "../app-framework"; import { webapp_client } from "../webapp-client"; -import * as prom_client from "../prom-client"; export function init_ping(): void { - let prom_ping_time: any = undefined, - prom_ping_time_last: any = undefined; - if (prom_client.enabled) { - prom_ping_time = prom_client.new_histogram("ping_ms", "ping time", { - buckets: [50, 100, 150, 200, 300, 500, 1000, 2000, 5000], - }); - prom_ping_time_last = prom_client.new_gauge( - "ping_last_ms", - "last reported ping time" - ); - } - webapp_client.on("ping", (ping_time: number): void => { let ping_time_smooth = redux.getStore("page").get("avgping") ?? ping_time; @@ -34,10 +21,5 @@ export function init_ping(): void { ping_time_smooth = decay * ping_time_smooth + (1 - decay) * ping_time; } redux.getActions("page").set_ping(ping_time, Math.round(ping_time_smooth)); - - if (prom_client.enabled) { - prom_ping_time?.observe(ping_time); - prom_ping_time_last?.set(ping_time); - } }); } diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 20966c6502..9533d98425 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -26,7 +26,6 @@ import { NatsClient } from "./nats"; import { HubClient } from "./hub"; import { IdleClient } from "./idle"; import { version } from "@cocalc/util/smc-version"; -import { start_metrics } from "../prom-client"; import { setup_global_cocalc } from "./console"; import { Query } from "@cocalc/sync/table"; import debug from "debug"; @@ -296,7 +295,6 @@ class Client extends EventEmitter implements WebappClient { this.time_client.ping(); // this will ping periodically }); - this.init_prom_client(); this.init_global_cocalc(); bind_methods(this); @@ -307,10 +305,6 @@ class Client extends EventEmitter implements WebappClient { setup_global_cocalc(this); } - private init_prom_client(): void { - this.on("start_metrics", start_metrics); - } - public dbg(f): Function { if (log.enabled) { return (...args) => log(new Date().toISOString(), f, ...args); diff --git a/src/packages/frontend/client/console.ts b/src/packages/frontend/client/console.ts index cce4f183ff..4043fe2f47 100644 --- a/src/packages/frontend/client/console.ts +++ b/src/packages/frontend/client/console.ts @@ -48,7 +48,6 @@ export function setup_global_cocalc(client): void { cocalc.misc = require("@cocalc/util/misc"); cocalc.immutable = require("immutable"); cocalc.done = cocalc.misc.done; - cocalc.prom_client = require("../prom-client"); cocalc.schema = require("@cocalc/util/schema"); cocalc.redux = redux; cocalc.load_eruda = load_eruda; diff --git a/src/packages/frontend/client/hub.ts b/src/packages/frontend/client/hub.ts index a6e16ea077..2771732a4b 100644 --- a/src/packages/frontend/client/hub.ts +++ b/src/packages/frontend/client/hub.ts @@ -225,9 +225,6 @@ export class HubClient { } break; - case "start_metrics": - this.client.emit("start_metrics", mesg.interval_s); - break; } // the call f(null, mesg) below can mutate mesg (!), so we better save the id here. diff --git a/src/packages/frontend/client/tracking.ts b/src/packages/frontend/client/tracking.ts index 05465d25d2..3a8dc2b13b 100644 --- a/src/packages/frontend/client/tracking.ts +++ b/src/packages/frontend/client/tracking.ts @@ -16,13 +16,6 @@ export class TrackingClient { this.client = client; } - // Send metrics to the hub this client is connected to. - // There is no confirmation or response that this succeeded, - // which is fine, since dropping some metrics is fine. - send_metrics = (metrics: object): void => { - this.client.hub_client.send(message.metrics({ metrics })); - }; - user_tracking = async (event: string, value: object): Promise => { if (this.userTrackingEnabled == null) { this.userTrackingEnabled = redux diff --git a/src/packages/frontend/last.ts b/src/packages/frontend/last.ts index aa0192df82..bc918220a0 100644 --- a/src/packages/frontend/last.ts +++ b/src/packages/frontend/last.ts @@ -11,8 +11,6 @@ declare var COCALC_GIT_REVISION: string; import { webapp_client } from "./webapp-client"; import { wrap_log } from "@cocalc/util/misc"; -import { get_browser, IS_MOBILE, IS_TOUCH } from "./feature"; -import * as prom_client from "./prom-client"; // import this specifically to cause th import checkFeaturesEnabled from "@cocalc/frontend/misc/check-features-enabled"; @@ -49,26 +47,6 @@ export function init() { // enable logging wrap_log(); - // finally, record start time - // TODO compute and report startup initialization time - if (prom_client.enabled) { - const browser_info_gauge = prom_client.new_gauge( - "browser_info", - "Information about the browser", - ["browser", "mobile", "touch", "git_version"] - ); - browser_info_gauge - .labels(get_browser(), IS_MOBILE, IS_TOUCH, COCALC_GIT_REVISION ?? "N/A") - .set(1); - const initialization_time_gauge = prom_client.new_gauge( - "initialization_seconds", - "Time from loading app.html page until last.coffee is completely done" - ); - initialization_time_gauge.set( - (new Date().getTime() - (window as any).webapp_initial_start_time) / 1000 - ); - } - // check for localStorage, etc. checkFeaturesEnabled(); } diff --git a/src/packages/frontend/package.json b/src/packages/frontend/package.json index 6a8ab110fe..7422b89439 100644 --- a/src/packages/frontend/package.json +++ b/src/packages/frontend/package.json @@ -140,7 +140,6 @@ "pica": "^7.1.0", "plotly.js": "^2.29.1", "project-name-generator": "^2.1.6", - "prom-client": "^13.0.0", "prop-types": "^15.7.2", "re-resizable": "^6.9.0", "react": "^18.3.1", diff --git a/src/packages/frontend/project/directory-listing.ts b/src/packages/frontend/project/directory-listing.ts index c5f227f6dd..4473b70e7d 100644 --- a/src/packages/frontend/project/directory-listing.ts +++ b/src/packages/frontend/project/directory-listing.ts @@ -7,24 +7,11 @@ import { server_time } from "@cocalc/util/misc"; import { once, retry_until_success } from "@cocalc/util/async-utils"; import { webapp_client } from "../webapp-client"; import { redux } from "../app-framework"; -import * as prom_client from "../prom-client"; import { dirname } from "path"; //const log = (...args) => console.log("directory-listing", ...args); const log = (..._args) => {}; -let prom_get_dir_listing_h; -if (prom_client.enabled) { - prom_get_dir_listing_h = prom_client.new_histogram( - "get_dir_listing_seconds", - "get_directory_listing time", - { - buckets: [1, 2, 5, 7, 10, 15, 20, 30, 50], - labels: ["public", "state", "err"], - }, - ); -} - interface ListingOpts { project_id: string; path: string; @@ -39,11 +26,6 @@ interface ListingOpts { export async function get_directory_listing(opts: ListingOpts) { log("get_directory_listing", opts); - let prom_dir_listing_start, prom_labels; - if (prom_client.enabled) { - prom_dir_listing_start = server_time(); - prom_labels = { public: false }; - } let method, state, time0, timeout; @@ -56,9 +38,6 @@ export async function get_directory_listing(opts: ListingOpts) { "state", "state", ]); - if (prom_client.enabled) { - prom_labels.state = state; - } if (state != null && state !== "running") { timeout = 0.5; time0 = server_time(); @@ -114,17 +93,6 @@ export async function get_directory_listing(opts: ListingOpts) { } catch (err) { listing_err = err; } finally { - if (prom_client.enabled && prom_dir_listing_start != null) { - prom_labels.err = !!listing_err; - const tm = - (server_time().valueOf() - prom_dir_listing_start.valueOf()) / 1000; - if (!isNaN(tm)) { - if (prom_get_dir_listing_h != null) { - prom_get_dir_listing_h.observe(prom_labels, tm); - } - } - } - // no error, but `listing` has no value, too // https://github.com/sagemathinc/cocalc/issues/3223 if (!listing_err && listing == null) { diff --git a/src/packages/frontend/prom-client.ts b/src/packages/frontend/prom-client.ts deleted file mode 100644 index 3c29f3c48a..0000000000 --- a/src/packages/frontend/prom-client.ts +++ /dev/null @@ -1,123 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Use prom-client in browser! - -NOTE: We explicitly import inside the prom-client package, since the index.js -in that package imports some things that make no sense in a browser. -*/ - -const PREFIX = "webapp_"; - -import { COCALC_MINIMAL } from "./fullscreen"; -export const enabled = true && !COCALC_MINIMAL; -console.log("initializing prometheus client. enabled =", enabled); - -import { webapp_client } from "./webapp-client"; - -import { globalRegistry } from "prom-client/lib/registry"; -import Counter from "prom-client/lib/counter"; -import Gauge from "prom-client/lib/gauge"; -import Histogram from "prom-client/lib/histogram"; -import Summary from "prom-client/lib/summary"; - -// ATTN: default metrics do not work, because they are only added upon "proper" export -- not our .get json trick -// register.setDefaultLabels(defaultLabels) - -async function send() { - if (!webapp_client.is_connected()) { - //console.log("prom-client.send: not connected") - return; - } - const metrics = await globalRegistry.getMetricsAsJSON(); - return webapp_client.tracking_client.send_metrics(metrics); -} - -let _interval_s: ReturnType | undefined = undefined; - -export async function start_metrics(interval_s = 120) { - //console.log('start_metrics') - stop_metrics(); - // send once so hub at least knows something about our metrics. - await send(); - // and then send every interval_s seconds: - return (_interval_s = setInterval(send, 1000 * interval_s)); -} - -function stop_metrics() { - if (_interval_s != null) { - clearInterval(_interval_s); - return (_interval_s = undefined); - } -} - -// a prometheus counter -- https://github.com/siimon/prom-client#counter -// usage: counter.labels(labelA, labelB).inc([positive number or default is 1]) -export function new_counter(name: string, help: string, labels: string[] = []) { - if (!name.endsWith("_total")) { - throw `Counter metric names have to end in [_unit]_total but I got '${name}' -- https://prometheus.io/docs/practices/naming/`; - } - return new Counter({ name: PREFIX + name, help, labelNames: labels }); -} - -// a prometheus gauge -- https://github.com/siimon/prom-client#gauge -// usage: gauge.labels(labelA, labelB).set(value) -export function new_gauge(name: string, help: string, labels: string[] = []) { - return new Gauge({ name: PREFIX + name, help, labelNames: labels }); -} - -interface QuantileConfig { - percentiles?: number[]; - labels?: string[]; -} - -// invoked as quantile.observe(value) -export function new_quantile( - name: string, - help: string, - config: QuantileConfig = {} -) { - config = { - ...{ - // a few more than the default, in particular including the actual min and max - percentiles: [0.0, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 0.999, 1.0], - labels: [], - }, - ...config, - }; - return new Summary({ - name: PREFIX + name, - help, - labelNames: config.labels, - percentiles: config.percentiles, - }); -} - -interface HistogramConfig { - buckets?: number[]; - labels?: string[]; -} - -// invoked as histogram.observe(value) -export function new_histogram( - name: string, - help: string, - config: HistogramConfig = {} -) { - config = { - ...{ - buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10], - labels: [], - }, - ...config, - }; - return new Histogram({ - name: PREFIX + name, - help, - labelNames: config.labels, - buckets: config.buckets, - }); -} diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee index 8a1cf2348c..49ee69813b 100644 --- a/src/packages/hub/client.coffee +++ b/src/packages/hub/client.coffee @@ -95,27 +95,6 @@ CLIENT_DESTROY_TIMER_S = 60*10 # 10 minutes CLIENT_MIN_ACTIVE_S = 45 -# How frequently we tell the browser clients to report metrics back to us. -# Set to 0 to completely disable metrics collection from clients. -CLIENT_METRICS_INTERVAL_S = if DEBUG2 then 15 else 60*2 - -# recording metrics and statistics -metrics_recorder = require('./metrics-recorder') - -# setting up client metrics -mesg_from_client_total = metrics_recorder.new_counter('mesg_from_client_total', - 'counts Client::handle_json_message_from_client invocations', ['event']) -push_to_client_stats_h = metrics_recorder.new_histogram('push_to_client_histo_ms', 'Client: push_to_client', - buckets : [1, 10, 50, 100, 200, 500, 1000, 2000, 5000, 10000] - labels: ['event'] - ) - -# All known metrics from connected clients. (Map from id to metrics.) -# id is deleted from this when client disconnects. -client_metrics = metrics_recorder.client_metrics - -if not misc.is_object(client_metrics) - throw Error("metrics_recorder must have a client_metrics attribute map") class exports.Client extends EventEmitter constructor: (opts) -> @@ -182,9 +161,6 @@ class exports.Client extends EventEmitter # and this fails, user gets a message, and see that they must sign in. @_remember_me_interval = setInterval(@check_for_remember_me, 1000*60*5) - if CLIENT_METRICS_INTERVAL_S - @push_to_client(message.start_metrics(interval_s:CLIENT_METRICS_INTERVAL_S)) - touch: (opts={}) => if not @account_id # not logged in opts.cb?('not logged in') @@ -255,9 +231,7 @@ class exports.Client extends EventEmitter @database.cancel_user_queries(client_id:@id) delete @_project_cache - delete client_metrics[@id] clearInterval(@_remember_me_interval) - @query_cancel_all_changefeeds() @closed = true @emit('close') @compute_session_uuids = [] @@ -360,7 +334,6 @@ class exports.Client extends EventEmitter @_messages.count += 1 avg = Math.round(@_messages.total_time / @_messages.count) dbg("[#{time_taken} mesg_time_ms] [#{avg} mesg_avg_ms] -- mesg.id=#{mesg.id}") - push_to_client_stats_h.observe({event:mesg.event}, time_taken) # If cb *is* given and mesg.id is *not* defined, then # we also setup a listener for a response from the client. @@ -597,7 +570,6 @@ class exports.Client extends EventEmitter # handler *should* handle any possible error, but just in case something # not expected goes wrong... we do this @error_to_client(id:mesg.id, error:"${err}") - mesg_from_client_total.labels("#{mesg.event}").inc(1) else @push_to_client(message.error(error:"Hub does not know how to handle a '#{mesg.event}' event.", id:mesg.id)) if mesg.event == 'get_all_activity' @@ -1703,36 +1675,6 @@ class exports.Client extends EventEmitter else @push_to_client(message.success(id:mesg.id)) - # Receive and store in memory the latest metrics status from the client. - mesg_metrics: (mesg) => - dbg = @dbg('mesg_metrics') - dbg() - if not mesg?.metrics - return - metrics = mesg.metrics - #dbg('GOT: ', misc.to_json(metrics)) - if not misc.is_array(metrics) - # client is messing with us...? - return - for metric in metrics - if not misc.is_array(metric?.values) - # what? - return - if metric.values.length == 0 - return - for v in metric.values - if not misc.is_object(v?.labels) - # what? - return - switch metric.type - when 'gauge' - metric.aggregator = 'average' - else - metric.aggregator = 'sum' - - client_metrics[@id] = metrics - #dbg('RECORDED: ', misc.to_json(client_metrics[@id])) - _check_project_access: (project_id, cb) => if not @account_id? cb('you must be signed in to access project') diff --git a/src/packages/hub/metrics-recorder.coffee b/src/packages/hub/metrics-recorder.coffee deleted file mode 100644 index aaacdbd251..0000000000 --- a/src/packages/hub/metrics-recorder.coffee +++ /dev/null @@ -1,181 +0,0 @@ -######################################################################### -# This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -# License: MS-RSL – see LICENSE.md for details -######################################################################### - -# This is a small helper class to record real-time metrics about the hub. -# It is designed for the hub, such that a local process can easily check its health. -# After an initial version, this has been repurposed to use prometheus. -# It wraps its client elements and adds some instrumentation to some hub components. - -fs = require('fs') -path = require('path') -underscore = require('underscore') -{execSync} = require('child_process') -{defaults} = misc = require('@cocalc/util/misc') - -# Prometheus client setup -- https://github.com/siimon/prom-client -prom_client = require('prom-client') - -# some constants -FREQ_s = 5 # update stats every FREQ seconds -DELAY_s = 10 # with an initial delay of DELAY seconds - -# collect some recommended default metrics -prom_client.collectDefaultMetrics(timeout: FREQ_s * 1000) - -# CLK_TCK (usually 100, but maybe not ...) -try - CLK_TCK = parseInt(execSync('getconf CLK_TCK', {encoding: 'utf8'})) -catch err - CLK_TCK = null - -### -# there is more than just continuous values -# cont: continuous (like number of changefeeds), will be smoothed -# disc: discrete, like blocked, will be recorded with timestamp -# in a queue of length DISC_LEN -exports.TYPE = TYPE = - COUNT: 'counter' # strictly non-decrasing integer - GAUGE: 'gauge' # only the most recent value is recorded - LAST : 'latest' # only the most recent value is recorded - DISC : 'discrete' # timeseries of length DISC_LEN - CONT : 'continuous' # continuous with exponential decay - MAX : 'contmax' # like CONT, reduces buffer to max value - SUM : 'contsum' # like CONT, reduces buffer to sum of values divided by FREQ_s -### - -PREFIX = 'cocalc_hub_' - -exports.new_counter = new_counter = (name, help, labels) -> - # a prometheus counter -- https://github.com/siimon/prom-client#counter - # use it like counter.labels(labelA, labelB).inc([positive number or default is 1]) - if not name.endsWith('_total') - throw "Counter metric names have to end in [_unit]_total but I got '#{name}' -- https://prometheus.io/docs/practices/naming/" - return new prom_client.Counter(name: PREFIX + name, help: help, labelNames: labels ? []) - -exports.new_gauge = new_gauge = (name, help, labels) -> - # a prometheus gauge -- https://github.com/siimon/prom-client#gauge - # basically, use it like gauge.labels(labelA, labelB).set(value) - return new prom_client.Gauge(name: PREFIX + name, help: help, labelNames: labels ? []) - -exports.new_quantile = new_quantile = (name, help, config={}) -> - # invoked as quantile.observe(value) - config = defaults config, - # a few more than the default, in particular including the actual min and max - percentiles: [0.0, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 0.999, 1.0] - labels : [] - return new prom_client.Summary(name: PREFIX + name, help: help, labelNames:config.labels, percentiles: config.percentiles) - -exports.new_histogram = new_histogram = (name, help, config={}) -> - # invoked as histogram.observe(value) - config = defaults config, - buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] - labels: [] - return new prom_client.Histogram(name: PREFIX + name, help: help, labelNames: config.labels, buckets:config.buckets) - - -# This is modified by the Client class (in client.coffee) when metrics -# get pushed from browsers. It's a map from client_id to -# an array of metrics objects, which are already labeled with extra -# information about the client_id and account_id. -exports.client_metrics = {} - -class MetricsRecorder - constructor: (@dbg, cb) -> - ### - * @dbg: reporting via winston, instance with configuration passed in from hub.coffee - ### - # stores the current state of the statistics - @_stats = {} - @_types = {} # key → TYPE.T mapping - - # the full statistic - @_data = {} - @_collectors = [] - - # initialization finished - @setup_monitoring() - cb?(undefined, @) - - client_metrics: => - ### - exports.client_metrics is a mapping of client id to the json exported metric. - The AggregatorRegistry is supposed to work with a list of metrics, and by default, - it sums them up. `aggregate` is a static method and hence it should be ok to use it directly. - ### - metrics = (m for _, m of exports.client_metrics) - - registry = prom_client.AggregatorRegistry.aggregate(metrics) - return await registry.metrics() - - metrics: => - ### - get a serialized representation of the metrics status - (was a dict that should be JSON, now it is for prometheus) - it's only called by the HTTP stuff in servers for the /metrics endpoint - ### - hub = await prom_client.register.metrics() - clients = await @client_metrics() - return hub + clients - - register_collector: (collector) => - # The added collector functions will be evaluated periodically to gather metrics - @_collectors.push(collector) - - setup_monitoring: => - # setup monitoring of some components - # called by the hub *after* setting up the DB, etc. - num_clients_gauge = new_gauge('clients_count', 'Number of connected clients') - {number_of_clients} = require('./hub_register') - @register_collector -> - try - num_clients_gauge.set(number_of_clients()) - catch - num_clients_gauge.set(0) - - - # our own CPU metrics monitor, separating user and sys! - # it's actually a counter, since it is non-decreasing, but we'll use .set(...) - @_cpu_seconds_total = new_gauge('process_cpu_categorized_seconds_total', 'Total number of CPU seconds used', ['type']) - - @_collect_duration = new_histogram('metrics_collect_duration_s', 'How long it took to gather the metrics', buckets:[0.0001, 0.001, 0.01, 1]) - @_collect_duration_last = new_gauge('metrics_collect_duration_s_last', 'How long it took the last time to gather the metrics') - - # init periodically calling @_collect - setTimeout((=> setInterval(@_collect, FREQ_s * 1000)), DELAY_s * 1000) - - _collect: => - endG = @_collect_duration_last.startTimer() - endH = @_collect_duration.startTimer() - - # called by @_update to evaluate the collector functions - #@dbg('_collect called') - for c in @_collectors - c() - # linux specific: collecting this process and all its children sys+user times - # http://man7.org/linux/man-pages/man5/proc.5.html - fs.readFile path.join('/proc', ''+process.pid, 'stat'), 'utf8', (err, infos) => - if err or not CLK_TCK? - @dbg("_collect err: #{err}") - return - # there might be spaces in the process name, hence split after the closing bracket! - infos = infos[infos.lastIndexOf(')') + 2...].split(' ') - @_cpu_seconds_total.labels('user') .set(parseFloat(infos[11]) / CLK_TCK) - @_cpu_seconds_total.labels('system') .set(parseFloat(infos[12]) / CLK_TCK) - # time spent waiting on child processes - @_cpu_seconds_total.labels('chld_user') .set(parseFloat(infos[13]) / CLK_TCK) - @_cpu_seconds_total.labels('chld_system').set(parseFloat(infos[14]) / CLK_TCK) - - # END: the timings for this run. - endG() - endH() - -metricsRecorder = null -exports.init = (winston, cb) -> - dbg = (msg) -> - winston.info("MetricsRecorder: #{msg}") - metricsRecorder = new MetricsRecorder(dbg, cb) - -exports.get = -> - return metricsRecorder diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index c7be875583..5ec69023ea 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -559,9 +559,6 @@ importers: project-name-generator: specifier: ^2.1.6 version: 2.1.9 - prom-client: - specifier: ^13.0.0 - version: 13.2.0 prop-types: specifier: ^15.7.2 version: 15.8.1 diff --git a/src/packages/server/nats/api/system.ts b/src/packages/server/nats/api/system.ts index 3f89af0fed..61dffbe6af 100644 --- a/src/packages/server/nats/api/system.ts +++ b/src/packages/server/nats/api/system.ts @@ -22,3 +22,4 @@ export async function userTracking({ }): Promise { await record_user_tracking(db(), account_id!, event, value); } + diff --git a/src/packages/util/message.d.ts b/src/packages/util/message.d.ts index 6b2470946c..8a56777c67 100644 --- a/src/packages/util/message.d.ts +++ b/src/packages/util/message.d.ts @@ -99,8 +99,6 @@ export const api_keys: any; export const api_keys_response: any; export const user_auth: any; export const user_auth_token: any; -export const metrics: any; -export const start_metrics: any; export const get_available_upgrades: any; export const disconnect_from_project: any; export const available_upgrades: any; diff --git a/src/packages/util/message.js b/src/packages/util/message.js index 1d17d1a5c3..261334d39f 100644 --- a/src/packages/util/message.js +++ b/src/packages/util/message.js @@ -2633,26 +2633,6 @@ Revoke a temporary authentication token for an account. """ */ -// client --> hub -message2({ - event: "metrics", - fields: { - metrics: { - init: required, - desc: "object containing the metrics", - }, - }, -}); - -message2({ - event: "start_metrics", - fields: { - interval_s: { - init: required, - desc: "tells client that it should submit metrics to the hub every interval_s seconds", - }, - }, -}); // Info about available upgrades for a given user API( From 5e8856eaef6e8b612627018bd86b89908db33191 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 3 Feb 2025 04:10:32 +0000 Subject: [PATCH 104/281] nats/api: only use new api for managing api keys... and - deleted all the old "legacy api key" code, which was coffeescript or autotranslated typescript, and probably dangerous. This makes cocalc more secure. - deeply entangled in there was some tracking of sign-ins for SSO. I just deleted all of that. I never look at those anyways and we could add back something better if it matters. --- .../database/settings/auth-sso-types.ts | 11 - src/packages/frontend/admin/json-editor.tsx | 4 +- src/packages/frontend/client/account.ts | 27 +- src/packages/frontend/client/project.ts | 15 +- .../codemirror/extensions/ai-formula.tsx | 2 +- src/packages/frontend/components/api-keys.tsx | 9 +- src/packages/hub/client.coffee | 45 --- src/packages/nats/hub-api/system.ts | 15 + src/packages/next/pages/api/v2/api-key.ts | 21 -- src/packages/server/api/manage.ts | 71 +--- .../server/auth/sso/passport-login.ts | 60 ---- src/packages/server/hub/auth.ts | 2 - src/packages/server/hub/sign-in.ts | 306 ------------------ src/packages/server/nats/api/system.ts | 3 +- src/packages/util/db-schema/api-keys.ts | 2 + 15 files changed, 34 insertions(+), 559 deletions(-) delete mode 100644 src/packages/next/pages/api/v2/api-key.ts delete mode 100644 src/packages/server/hub/sign-in.ts diff --git a/src/packages/database/settings/auth-sso-types.ts b/src/packages/database/settings/auth-sso-types.ts index 63f938092d..65da3f916b 100644 --- a/src/packages/database/settings/auth-sso-types.ts +++ b/src/packages/database/settings/auth-sso-types.ts @@ -5,21 +5,10 @@ import { PostgreSQL } from "@cocalc/database/postgres/types"; -// see @hub/sign-in -interface RecordSignInOpts { - ip_address: string; - successful: boolean; - database: PostgreSQL; - email_address?: string; - account_id?: string; - remember_me: boolean; -} - export interface PassportLoginOpts { passports: { [k: string]: PassportStrategyDB }; database: PostgreSQL; strategyName: string; - record_sign_in: (opts: RecordSignInOpts) => void; // a function of that old "hub/sign-in" module profile: any; // complex object id: string; // id is required. e.g. take the email address – see create_passport in postgres-server-queries.coffee first_name?: string; diff --git a/src/packages/frontend/admin/json-editor.tsx b/src/packages/frontend/admin/json-editor.tsx index 5aa9760629..dd8a4084db 100644 --- a/src/packages/frontend/admin/json-editor.tsx +++ b/src/packages/frontend/admin/json-editor.tsx @@ -40,7 +40,7 @@ export const JsonEditor: React.FC = (props: Props) => { setError(""); if (save) onSave(oneLine); } catch (err) { - setError(err.message); + setError(`${err}`); } } @@ -48,7 +48,7 @@ export const JsonEditor: React.FC = (props: Props) => { try { setEditing(JSON.stringify(jsonic(editing), null, 2)); } catch (err) { - setError(err.message); + setError(`${err}`); } } diff --git a/src/packages/frontend/client/account.ts b/src/packages/frontend/client/account.ts index ae7af7dd84..5f8bc43b25 100644 --- a/src/packages/frontend/client/account.ts +++ b/src/packages/frontend/client/account.ts @@ -173,24 +173,6 @@ export class AccountClient { ); } - // legacy api: getting, setting, deleting, etc., the api key for this account - public async api_key( - action: "get" | "delete" | "regenerate", - password: string, - ): Promise { - if (this.client.account_id == null) { - throw Error("must be logged in"); - } - return ( - await this.call( - message.api_key({ - action, - password, - }), - ) - ).api_key; - } - // new interface: getting, setting, editing, deleting, etc., the api keys for a project public async api_keys(opts: { action: "get" | "delete" | "create" | "edit"; @@ -199,13 +181,6 @@ export class AccountClient { id?: number; expire?: Date; }): Promise { - if (this.client.account_id == null) { - throw Error("must be logged in"); - } - // because message always uses id, so we have to use something else! - const opts2: any = { ...opts }; - delete opts2.id; - opts2.key_id = opts.id; - return (await this.call(message.api_keys(opts2))).response; + return await this.client.nats_client.hub.system.manageApiKeys(opts); } } diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index b3da21bfd9..f9b83c56e4 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -613,20 +613,7 @@ export class ProjectClient { id?: number; expire?: Date; }): Promise { - if (this.client.account_id == null) { - throw Error("must be logged in"); - } - if (!is_valid_uuid_string(opts.project_id)) { - throw Error("project_id must be a valid uuid"); - } - if (opts.project_id == null && !opts.password) { - throw Error("must provide password for non-project api key"); - } - // because message always uses id, so we have to use something else! - const opts2: any = { ...opts }; - delete opts2.id; - opts2.key_id = opts.id; - return (await this.call(message.api_keys(opts2))).response; + return await this.client.nats_client.hub.system.manageApiKeys(opts); } computeServers = (project_id) => { diff --git a/src/packages/frontend/codemirror/extensions/ai-formula.tsx b/src/packages/frontend/codemirror/extensions/ai-formula.tsx index b47c7ba391..45afbd14c3 100644 --- a/src/packages/frontend/codemirror/extensions/ai-formula.tsx +++ b/src/packages/frontend/codemirror/extensions/ai-formula.tsx @@ -242,7 +242,7 @@ function AiGenFormula({ mode, text = "", project_id, locale, cb }: Props) { setFullReply(""); } } catch (err) { - setError(err.message || err.toString()); + setError(`${err}`); } finally { setGenerating(false); } diff --git a/src/packages/frontend/components/api-keys.tsx b/src/packages/frontend/components/api-keys.tsx index 41870daa79..ab1d75de64 100644 --- a/src/packages/frontend/components/api-keys.tsx +++ b/src/packages/frontend/components/api-keys.tsx @@ -78,7 +78,7 @@ export default function ApiKeys({ manage, mode = "project" }: Props) { setError(null); } catch (err) { setLoading(false); - setError(err.message || "An error occurred"); + setError(`${err}`); } }; @@ -87,7 +87,7 @@ export default function ApiKeys({ manage, mode = "project" }: Props) { await manage({ action: "delete", id }); getAllApiKeys(); } catch (err) { - setError(err.message || "An error occurred"); + setError(`${err}`); } }; @@ -102,7 +102,7 @@ export default function ApiKeys({ manage, mode = "project" }: Props) { await manage({ action: "edit", id, name, expire }); getAllApiKeys(); } catch (err) { - setError(err.message || "An error occurred"); + setError(`${err}`); } }; @@ -117,6 +117,7 @@ export default function ApiKeys({ manage, mode = "project" }: Props) { getAllApiKeys(); Modal.success({ + width: 600, title: "New Secret API Key", content: ( <> @@ -137,7 +138,7 @@ export default function ApiKeys({ manage, mode = "project" }: Props) { }); setError(null); } catch (err) { - setError(err.message || "An error occurred"); + setError(`${err}`); } }; diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee index 49ee69813b..3f55e944cc 100644 --- a/src/packages/hub/client.coffee +++ b/src/packages/hub/client.coffee @@ -19,12 +19,9 @@ clients = require('./clients').getClients() auth = require('./auth') auth_token = require('./auth-token') local_hub_connection = require('./local_hub_connection') -sign_in = require('@cocalc/server/hub/sign-in') hub_projects = require('./projects') {StripeClient} = require('@cocalc/server/stripe/client') {send_email, send_invite_email} = require('./email') -manageApiKeys = require("@cocalc/server/api/manage").default -{legacyManageApiKey} = require("@cocalc/server/api/manage") purchase_license = require('@cocalc/server/licenses/purchase').default db_schema = require('@cocalc/util/db-schema') { escapeHtml } = require("escape-html") @@ -394,12 +391,6 @@ class exports.Client extends EventEmitter # Record that this connection is authenticated as user with given uuid. @account_id = signed_in_mesg.account_id - sign_in.record_sign_in - ip_address : @ip_address - successful : true - account_id : signed_in_mesg.account_id - database : @database - # Get user's group from database. @get_groups() @@ -576,15 +567,6 @@ class exports.Client extends EventEmitter dbg("ignoring all further messages from old client=#{@id}") @_ignore_client = true - mesg_sign_in: (mesg) => - sign_in.sign_in - client : @ - mesg : mesg - logger : @logger - database : @database - host : @_opts.host - port : @_opts.port - mesg_sign_out: (mesg) => if not @account_id? @push_to_client(message.error(id:mesg.id, error:"not signed in")) @@ -1625,33 +1607,6 @@ class exports.Client extends EventEmitter # END stripe-related functionality - mesg_api_key: (mesg) => - try - api_key = await legacyManageApiKey - account_id : @account_id - password : mesg.password - action : mesg.action - if api_key - @push_to_client(message.api_key_info(id:mesg.id, api_key:api_key)) - else - @success_to_client(id:mesg.id) - catch err - @error_to_client(id:mesg.id, error:err) - - mesg_api_keys: (mesg) => - try - response = await manageApiKeys - account_id : @account_id - password : mesg.password - action : mesg.action - project_id : mesg.project_id - id : mesg.key_id - expire : mesg.expire - name : mesg.name - @push_to_client(message.api_keys_response(id:mesg.id, response:response)) - catch err - @error_to_client(id:mesg.id, error:err) - mesg_user_auth: (mesg) => auth_token.get_user_auth_token database : @database diff --git a/src/packages/nats/hub-api/system.ts b/src/packages/nats/hub-api/system.ts index 1dcf4f5698..a38ad79fa5 100644 --- a/src/packages/nats/hub-api/system.ts +++ b/src/packages/nats/hub-api/system.ts @@ -1,5 +1,9 @@ import { noAuth, authFirst } from "./util"; import type { Customize } from "@cocalc/util/db-schema/server-settings"; +import type { + ApiKey, + Action as ApiKeyAction, +} from "@cocalc/util/db-schema/api-keys"; export const system = { getCustomize: noAuth, @@ -7,6 +11,7 @@ export const system = { addProjectPermission: authFirst, terminate: authFirst, userTracking: authFirst, + manageApiKeys: authFirst, }; export interface System { @@ -20,9 +25,19 @@ export interface System { // - only admin can do this. // - useful for development terminate: (service: "database" | "api") => Promise; + userTracking: (opts: { event: string; value: object; account_id?: string; }) => Promise; + + manageApiKeys: (opts: { + account_id?: string; + action: ApiKeyAction; + project_id?: string; + name?: string; + expire?: Date; + id?: number; + }) => Promise; } diff --git a/src/packages/next/pages/api/v2/api-key.ts b/src/packages/next/pages/api/v2/api-key.ts deleted file mode 100644 index 12b2f5584e..0000000000 --- a/src/packages/next/pages/api/v2/api-key.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* -v2 API endpoint for managing your legacy API key. -*/ - -import getAccountId from "lib/account/get-account"; -import { legacyManageApiKey } from "@cocalc/server/api/manage"; -import getParams from "lib/api/get-params"; - -export default async function handle(req, res) { - try { - const account_id = await getAccountId(req); - if (!account_id) { - throw Error("must be signed in"); - } - const { action, password } = getParams(req); - const api_key = await legacyManageApiKey({ account_id, password, action }); - res.json({ api_key }); - } catch (err) { - res.json({ error: err.message }); - } -} diff --git a/src/packages/server/api/manage.ts b/src/packages/server/api/manage.ts index c12f565c8b..de810c4ec8 100644 --- a/src/packages/server/api/manage.ts +++ b/src/packages/server/api/manage.ts @@ -1,5 +1,4 @@ /* -User management of the v1 API key associated to an account. This supports three actions: - get: get the already created keys associated to an account or project @@ -12,8 +11,6 @@ they have no password, then the provided one is ignored. */ import getPool from "@cocalc/database/pool"; -import isPasswordCorrect from "@cocalc/server/auth/is-password-correct"; -import hasPassword from "@cocalc/server/auth/has-password"; import { generate } from "random-key"; import isCollaborator from "@cocalc/server/projects/is-collaborator"; import passwordHash, { @@ -22,7 +19,10 @@ import passwordHash, { import { getLogger } from "@cocalc/backend/logger"; import base62 from "base62/lib/ascii"; import isValidAccount from "@cocalc/server/accounts/is-valid-account"; -import type { ApiKey as ApiKeyType } from "@cocalc/util/db-schema/api-keys"; +import type { + ApiKey as ApiKeyType, + Action as ApiKeyAction, +} from "@cocalc/util/db-schema/api-keys"; import isBanned from "@cocalc/server/accounts/is-banned"; const log = getLogger("server:api:manage"); @@ -32,9 +32,6 @@ const log = getLogger("server:api:manage"); // to create 5K compute servers at once, don't make this too small. const MAX_API_KEYS = 100000; -// regenerate is only for the legacy api keys -type Action = "get" | "delete" | "regenerate" | "create" | "edit"; - // Converts any 32-bit nonnegative integer as a 6-character base-62 string. function encode62(n: number): string { if (!Number.isInteger(n)) { @@ -49,7 +46,7 @@ function decode62(s: string): number { interface Options { account_id: string; - action: Action; + action: ApiKeyAction; project_id?: string; name?: string; expire?: Date; @@ -334,61 +331,3 @@ export async function getAccountWithApiKey( return undefined; } -// Management of the OLD LEGACY API KEYS. - -export async function legacyManageApiKey({ - account_id, - password, - action, -}: { - account_id: string; - password?: string; - action: Action; -}): Promise { - // Check if the user has a password - if (await hasPassword(account_id)) { - if (!password) { - throw Error("password must be provided"); - } - // verify password is correct - if (!(await isPasswordCorrect({ account_id, password }))) { - throw Error("invalid password"); - } - } else if (!(await isValidAccount(account_id))) { - throw Error("account_id is not a valid account"); - } - - // Now we allow the action. - const pool = getPool(); - switch (action) { - case "get": - const { rows } = await pool.query( - "SELECT api_key FROM accounts WHERE account_id=$1::UUID", - [account_id], - ); - if (rows.length == 0) { - throw Error("no such account"); - } - return rows[0].api_key; - case "delete": - await pool.query( - "UPDATE accounts SET api_key=NULL WHERE account_id=$1::UUID", - [account_id], - ); - return; - case "regenerate": - // There is a unique index on api_key, so there is a small probability - // that this query fails. However, it's probably smaller than the probability - // that the database connection is broken, so if it were to ever happen, then - // the user could just retry. For context, for the last few years, this query - // happened on cocalc.com only a few thousand times *total*. - const api_key = `sk_${generate()}`; - await pool.query( - "UPDATE accounts SET api_key=$1 WHERE account_id=$2::UUID", - [api_key, account_id], - ); - return api_key; - default: - throw Error(`unknown action="${action}"`); - } -} diff --git a/src/packages/server/auth/sso/passport-login.ts b/src/packages/server/auth/sso/passport-login.ts index de3bb86b5e..3a5115a1ac 100644 --- a/src/packages/server/auth/sso/passport-login.ts +++ b/src/packages/server/auth/sso/passport-login.ts @@ -25,7 +25,6 @@ import base_path from "@cocalc/backend/base-path"; import getLogger from "@cocalc/backend/logger"; import { set_email_address_verified } from "@cocalc/database/postgres/account-queries"; import type { PostgreSQL } from "@cocalc/database/postgres/types"; -import { legacyManageApiKey } from "@cocalc/server/api/manage"; import generateHash from "@cocalc/server/auth/hash"; import { REMEMBER_ME_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; import { createRememberMeCookie } from "@cocalc/server/auth/remember-me"; @@ -54,7 +53,6 @@ export class PassportLogin { private readonly database: PostgreSQL; // passed on to do the login private opts: PassportLoginOpts; - private record_sign_in: Function; constructor(opts: PassportLoginOpts) { const L = logger.extend("constructor").debug; @@ -62,7 +60,6 @@ export class PassportLogin { this.passports = opts.passports; //this.exclusiveDomains = this.mapExclusiveDomains(); this.database = opts.database; - this.record_sign_in = opts.record_sign_in; this.opts = opts; @@ -138,12 +135,8 @@ export class PassportLogin { await this.checkExistingEmails(this.opts, locals); // if no account yet → create one await this.maybeCreateAccount(this.opts, locals); - // record a sign-in activity, if we deal with an existing account - await this.maybeRecordSignIn(this.opts, locals); // if update_on_login is true, update the account with the new profile data await this.maybeUpdateAccountAndPassport(this.opts, locals); - // deal with the case where user wants an API key - await this.maybeProvisionAPIKey(locals); // check if user is banned? await this.isUserBanned(locals.account_id, locals.email_address); // last step: set remember me cookie (for a new sign in) @@ -463,26 +456,6 @@ export class PassportLogin { }); } - // if the above created no new account (and hence we had an account_id before that) - // we record that we signed in a user - private async maybeRecordSignIn( - opts: PassportLoginOpts, - locals: PassportLoginLocals, - ): Promise { - if (locals.new_account_created) return; - const L = logger.extend("maybe_record_sign_in").debug; - - // don't make client wait for this -- it's just a log message for us. - L(`no new account → record_sign_in: ${opts.req.ip}`); - this.record_sign_in({ - ip_address: opts.req.ip, - successful: true, - remember_me: locals.has_valid_remember_me, - email_address: locals.email_address, - account_id: locals.account_id, - database: this.database, - }); - } // optionally, SSO strategies can be configured to always update fields of the user // with the data they provide. right now that's first and last name. @@ -515,39 +488,6 @@ export class PassportLogin { }); } - // There is a special case, where an api_key was requested. - // This is chekced here, key created, and the client is redirected to a special (local) URL - private async maybeProvisionAPIKey( - locals: PassportLoginLocals, - ): Promise { - if (!locals.get_api_key) return; - if (!locals.account_id) return; // typescript cares about this. - const L = logger.extend("maybe_provision_api_key").debug; - - // Just handle getting api key here. - if (locals.new_account_created) { - locals.action = "regenerate"; // obvious - } else { - locals.action = "get"; - } - - locals.api_key = await legacyManageApiKey({ - account_id: locals.account_id, - action: locals.action, - }); - - // if there is no key - if (!locals.api_key) { - L("must generate key, since don't already have it"); - locals.api_key = await legacyManageApiKey({ - account_id: locals.account_id, - action: "regenerate", - }); - } - // we got a key ... - // NOTE: See also code to generate similar URL in @cocalc/frontend/account/init.ts - locals.target = `https://authenticated?api_key=${locals.api_key}`; - } // ebfore recording the sign-in below, we check if a user is banned private async isUserBanned(account_id, email_address): Promise { diff --git a/src/packages/server/hub/auth.ts b/src/packages/server/hub/auth.ts index dae0d275cf..6cf19db6c4 100644 --- a/src/packages/server/hub/auth.ts +++ b/src/packages/server/hub/auth.ts @@ -98,7 +98,6 @@ import { GoogleStrategyConf, TwitterStrategyConf, } from "@cocalc/server/auth/sso/public-strategies"; -import { record_sign_in } from "./sign-in"; import { getServerSettings } from "@cocalc/database/settings"; import { signInUsingImpersonateToken } from "@cocalc/server/auth/impersonate"; @@ -626,7 +625,6 @@ export class PassportManager { passports: this.passports ?? {}, database: this.database, host: this.host, - record_sign_in, id: profile.id, // ATTN: not all strategies have an ID → you have to derive the ID from the profile below via the "login_info" mapping (e.g. {id: "email"}) strategyName: name, profile, // will just get saved in database diff --git a/src/packages/server/hub/sign-in.ts b/src/packages/server/hub/sign-in.ts deleted file mode 100644 index bbcbf8c95e..0000000000 --- a/src/packages/server/hub/sign-in.ts +++ /dev/null @@ -1,306 +0,0 @@ -//######################################################################## -// This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. -// License: MS-RSL – see LICENSE.md for details -//######################################################################## - -// WARNING: this is an ungly port from coffeescript of ancient code from the hub... - -/* -User sign in - -Throttling policy: It basically like this, except we reset the counters -each minute and hour, so a crafty attacker could get twice as many tries by finding the -reset interval and hitting us right before and after. This is an acceptable tradeoff -for making the data structure trivial. - - * POLICY 1: A given email address is allowed at most 3 failed login attempts per minute. - * POLICY 2: A given email address is allowed at most 30 failed login attempts per hour. - * POLICY 3: A given ip address is allowed at most 10 failed login attempts per minute. - * POLICY 4: A given ip address is allowed at most 50 failed login attempts per hour. -*/ - -import * as async from "async"; -import * as message from "@cocalc/util/message"; -import * as misc from "@cocalc/util/misc"; -const { required, defaults } = misc; -import * as auth from "./auth"; -import { process_env_int } from "@cocalc/backend/misc"; -import * as throttle from "@cocalc/server/auth/throttle"; -import Bottleneck from "bottleneck"; -import { legacyManageApiKey } from "@cocalc/server/api/manage"; - -// these parameters are per group and per hub! -const bottleneck_opts = { - minTime: process_env_int("THROTTLE_SIGN_IN_MINTIME", 100), - maxConcurrent: process_env_int("THROTTLE_SIGN_IN_CONCURRENT", 10), -}; -const limit_group = new Bottleneck.Group(bottleneck_opts); - -const record_sign_in_fail = function (opts) { - const { email, ip, logger } = defaults(opts, { - email: required, - ip: required, - logger: undefined, - }); - throttle.recordFail(email, ip); - return logger?.(`WARNING: record_sign_in_fail(${email}, ${ip})`); -}; - -const sign_in_check = function (opts) { - const { email, ip, auth_token } = defaults(opts, { - email: required, - ip: required, - auth_token: undefined, - }); - return throttle.signInCheck(email, ip, auth_token); -}; - -export function sign_in(opts) { - opts = defaults(opts, { - client: required, - mesg: required, - logger: undefined, - database: required, - host: undefined, - port: undefined, - cb: undefined, - }); - - const key = opts.client.ip_address; - - const done_cb = () => opts.logger?.debug(`sign_in(group='${key}'): done`); - - return limit_group.key(key).submit(_sign_in, opts, done_cb); -} - -async function _sign_in(opts, done) { - let dbg; - const { client, mesg } = opts; - if (opts.logger != null) { - dbg = (m) => opts.logger.debug(`_sign_in(${mesg.email_address}): ${m}`); - dbg(); - } else { - dbg = function () {}; - } - const tm = misc.walltime(); - - let signed_in_mesg: any = undefined; - let account: any = undefined; - - const sign_in_error = function (error) { - dbg(`sign_in_error -- ${error}`); - record_sign_in({ - database: opts.database, - ip_address: client.ip_address, - successful: false, - email_address: mesg.email_address, - account_id: account?.account_id, - }); - client.push_to_client( - message.sign_in_failed({ - id: mesg.id, - email_address: mesg.email_address, - reason: error, - }), - ); - return opts.cb?.(error); - }; - - if (!mesg.email_address) { - sign_in_error("Empty email address."); - return; - } - - if (!mesg.password) { - sign_in_error("Empty password."); - return; - } - - mesg.email_address = misc.lower_email_address(mesg.email_address); - - const m = await sign_in_check({ - email: mesg.email_address, - ip: client.ip_address, - auth_token: false, - }); - if (m) { - sign_in_error(`sign_in_check failure: ${m}`); - return; - } - - return async.series( - [ - function (cb) { - dbg("get account and check credentials"); - // NOTE: Despite people complaining, we do give away info about whether - // the e-mail address is for a valid user or not. - // There is no security in not doing this, since the same information - // can be determined via the invite collaborators feature. - return opts.database.get_account({ - email_address: mesg.email_address, - columns: ["password_hash", "account_id", "passports", "banned"], - cb(err, _account) { - if (err) { - cb(err); - return; - } - account = _account; - dbg(`account = ${JSON.stringify(account)}`); - if (account.banned) { - dbg("banned account!"); - cb( - Error( - "This account is BANNED. Contact help@cocalc.com if you believe this is a mistake.", - ), - ); - return; - } - return cb(); - }, - }); - }, - function (cb) { - dbg("got account; now checking if password is correct..."); - return auth.is_password_correct({ - database: opts.database, - account_id: account.account_id, - password: mesg.password, - password_hash: account.password_hash, - cb(err, is_correct) { - if (err) { - cb(Error(`Error checking correctness of password -- ${err}`)); - return; - } - if (!is_correct) { - if (!account.password_hash) { - return cb( - Error( - `The account ${ - mesg.email_address - } exists but doesn't have a password. Either set your password by clicking 'Forgot Password?' or log in using ${misc - .keys(account.passports) - .join( - ", ", - )}. If that doesn't work, email help@cocalc.com and we will sort this out.`, - ), - ); - } else { - return cb( - Error( - `Incorrect password for ${mesg.email_address}. You can reset your password by clicking the 'Forgot Password?' link. If that doesn't work, email help@cocalc.com and we will sort this out.`, - ), - ); - } - } else { - return cb(); - } - }, - }); - }, - // remember me - function (cb) { - if (!mesg.remember_me) { - // do not set cookie if already set (and message known) - cb(); - return; - } - dbg("remember_me -- setting the remember_me cookie"); - signed_in_mesg = message.signed_in({ - id: mesg.id, - account_id: account.account_id, - email_address: mesg.email_address, - remember_me: false, - hub: opts.host + ":" + opts.port, - }) as any; - client.remember_me({ - account_id: signed_in_mesg.account_id, - cb, - }); - }, - async function (cb) { - if (!mesg.get_api_key) { - cb(); - return; - } - dbg("get_api_key -- also get_api_key"); - try { - signed_in_mesg.api_key = await legacyManageApiKey({ - account_id: account.account_id, - password: mesg.password, - action: "get", - }); - return cb(); - } catch (err) { - return cb(err); - } - }, - async function (cb) { - if (!mesg.get_api_key || signed_in_mesg.api_key) { - cb(); - return; - } - dbg("get_api_key -- must generate key since don't already have it"); - try { - signed_in_mesg.api_key = await legacyManageApiKey({ - account_id: account.account_id, - password: mesg.password, - action: "regenerate", - }); - return cb(); - } catch (err) { - return cb(err); - } - }, - ], - function (err) { - if (err) { - dbg(`send error to user (in ${misc.walltime(tm)}seconds) -- ${err}`); - sign_in_error(err); - opts.cb?.(err); - } else { - dbg( - `user got signed in fine (in ${misc.walltime( - tm, - )}seconds) -- sending them a message`, - ); - client.signed_in(signed_in_mesg); - client.push_to_client(signed_in_mesg); - opts.cb?.(); - } - - // final callback for bottleneck group - return done(); - }, - ); -} - -// Record to the database a failed and/or successful login attempt. -export function record_sign_in(opts) { - opts = defaults(opts, { - ip_address: required, - successful: required, - database: required, - email_address: undefined, - account_id: undefined, - remember_me: false, - }); - - if (!opts.successful) { - return record_sign_in_fail({ - email: opts.email_address, - ip: opts.ip_address, - }); - } else { - const data = { - ip_address: opts.ip_address, - email_address: opts.email_address != null ? opts.email_address : null, - remember_me: opts.remember_me, - account_id: opts.account_id, - }; - - return opts.database.log({ - event: "successful_sign_in", - value: data, - }); - } -} diff --git a/src/packages/server/nats/api/system.ts b/src/packages/server/nats/api/system.ts index 61dffbe6af..4721021135 100644 --- a/src/packages/server/nats/api/system.ts +++ b/src/packages/server/nats/api/system.ts @@ -4,6 +4,8 @@ import { addProjectPermission } from "@cocalc/server/nats/auth"; export { addProjectPermission }; import { record_user_tracking } from "@cocalc/database/postgres/user-tracking"; import { db } from "@cocalc/database"; +import manageApiKeys from "@cocalc/server/api/manage"; +export { manageApiKeys }; export function ping() { return { now: Date.now() }; @@ -22,4 +24,3 @@ export async function userTracking({ }): Promise { await record_user_tracking(db(), account_id!, event, value); } - diff --git a/src/packages/util/db-schema/api-keys.ts b/src/packages/util/db-schema/api-keys.ts index 2c8c69e947..7e045c2a38 100644 --- a/src/packages/util/db-schema/api-keys.ts +++ b/src/packages/util/db-schema/api-keys.ts @@ -1,6 +1,8 @@ import { Table } from "./types"; import { CREATED_BY, ID } from "./crm"; +export type Action = "get" | "delete" | "create" | "edit"; + export interface ApiKey { id: number; account_id: string; From c12c036b4e195d61f90c92ff971236a91e028799 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 3 Feb 2025 04:18:47 +0000 Subject: [PATCH 105/281] delete ancient deprecated code for zendesk in the app --- src/packages/frontend/client/client.ts | 6 - src/packages/frontend/client/support.ts | 33 ----- src/packages/hub/client.coffee | 14 --- src/packages/util/message.d.ts | 4 - src/packages/util/message.js | 153 ------------------------ 5 files changed, 210 deletions(-) delete mode 100644 src/packages/frontend/client/support.ts diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 9533d98425..75175ef500 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -8,7 +8,6 @@ import { delay } from "awaiting"; import { alert_message } from "../alerts"; import { StripeClient } from "./stripe"; import { ProjectCollaborators } from "./project-collaborators"; -import { SupportTickets } from "./support"; import { Messages } from "./messages"; import { QueryClient } from "./query"; import { TimeClient } from "./time"; @@ -49,7 +48,6 @@ export interface WebappClient extends EventEmitter { stripe: StripeClient; project_collaborators: ProjectCollaborators; - support_tickets: SupportTickets; messages: Messages; query_client: QueryClient; time_client: TimeClient; @@ -131,7 +129,6 @@ class Client extends EventEmitter implements WebappClient { account_id?: string; stripe: StripeClient; project_collaborators: ProjectCollaborators; - support_tickets: SupportTickets; messages: Messages; query_client: QueryClient; time_client: TimeClient; @@ -211,9 +208,6 @@ class Client extends EventEmitter implements WebappClient { this.project_collaborators = bind_methods( new ProjectCollaborators(this.async_call.bind(this)), ); - this.support_tickets = bind_methods( - new SupportTickets(this.async_call.bind(this)), - ); this.messages = new Messages(); this.query_client = bind_methods(new QueryClient(this)); this.time_client = bind_methods(new TimeClient(this)); diff --git a/src/packages/frontend/client/support.ts b/src/packages/frontend/client/support.ts deleted file mode 100644 index b085a84c03..0000000000 --- a/src/packages/frontend/client/support.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { replace_all } from "@cocalc/util/misc"; -import * as message from "@cocalc/util/message"; -import { AsyncCall } from "./client"; - -export class SupportTickets { - private async_call: AsyncCall; - - constructor(async_call: AsyncCall) { - this.async_call = async_call; - } - - private async call(message: object): Promise { - return await this.async_call({ message, timeout: 30 }); - } - - public async create(opts): Promise { - if (opts.body != null) { - // Make it so the session is ignored in any URL appearing in the body. - // Obviously, this is not 100% bullet proof, but should help enormously. - opts.body = replace_all(opts.body, "?session=", "?session=#"); - } - return (await this.call(message.create_support_ticket(opts))).url; - } - - public async get(): Promise { - return (await this.call(message.get_support_tickets())).tickets; - } -} diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee index 3f55e944cc..cb7bf09942 100644 --- a/src/packages/hub/client.coffee +++ b/src/packages/hub/client.coffee @@ -1525,20 +1525,6 @@ class exports.Client extends EventEmitter ) - ### - Support Tickets → Zendesk - ### - mesg_create_support_ticket: (mesg) => - dbg = @dbg("mesg_create_support_ticket") - dbg('deprecated') - @error_to_client(id:mesg.id, error:'deprecated') - - mesg_get_support_tickets: (mesg) => - # retrieves the support tickets the user with the current account_id - dbg = @dbg("mesg_get_support_tickets") - dbg('deprecated') - @error_to_client(id:mesg.id, error:'deprecated') - ### Stripe-integration billing code ### diff --git a/src/packages/util/message.d.ts b/src/packages/util/message.d.ts index 8a56777c67..236a038c9a 100644 --- a/src/packages/util/message.d.ts +++ b/src/packages/util/message.d.ts @@ -85,10 +85,6 @@ export const stripe_charges: any; export const stripe_get_invoices: any; export const stripe_invoices: any; export const stripe_admin_create_invoice_item: any; -export const create_support_ticket: any; -export const support_ticket_url: any; -export const get_support_tickets: any; -export const support_tickets: any; export const query: any; export const uuid: any; export const uuid: any; diff --git a/src/packages/util/message.js b/src/packages/util/message.js index 261334d39f..abcf65dd33 100644 --- a/src/packages/util/message.js +++ b/src/packages/util/message.js @@ -2093,159 +2093,6 @@ message({ description: undefined, }); -/* -Support Tickets → right now going through Zendesk -*/ - -// client → hub -API( - message2({ - event: "create_support_ticket", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - username: { - init: undefined, - desc: "name on the ticket", - }, - email_address: { - init: required, - desc: "if there is no email_address in the account, there cannot be a ticket!", - }, - subject: { - init: required, - desc: "like an email subject", - }, - body: { - init: required, - desc: "html or md formatted text", - }, - tags: { - init: undefined, - desc: "a list of tags, like `['member']`", - }, - account_id: { - init: undefined, - desc: "account_id for the ticket", - }, - location: { - init: undefined, - desc: "from the URL, to know what the requester is talking about", - }, - info: { - init: undefined, - desc: "additional data dict, like browser/OS", - }, - }, - desc: `\ -Open a CoCalc support ticket. - -Notes: - -- If \`account_id\` is not provided, the ticket will be created, but ticket -info will not be returned by \`get_support_tickets\`. - -- If \`username\` is not provided, \`email_address\` is used for the name on the ticket. - -- \`location\` is used to provide a path to a specific project or file, for example - \`\`\` - /project/a17037cb-a083-4519-b3c1-38512af603a6/files/notebook.ipynb\` - \`\`\` - -If present, the \`location\` string will be expanded to a complete URL and -appended to the body of the ticket. - -- The \`info\` dict can be used to provide additional metadata, for example - \`\`\` - {"user_agent":"Mozilla/5.0 ... Chrome/58.0.3029.96 Safari/537.36"} - \`\`\` - -- If the ticket concerns a CoCalc course, the project id of the course can be included in the \`info\` dict, for example, - \`\`\` - {"course":"0c7ae00c-ea43-4981-b454-90d4a8b1ac47"} - \`\`\` - - In that case, the course project_id will be expanded to a URL and appended to the body of the ticket. - -- If \`tags\` or \`info\` are provided, options must be sent as a JSON object. - -Example: - -\`\`\` - curl -u sk_abcdefQWERTY090900000000: -H "Content-Type: application/json" \\ - -d '{"email_address":"jd@example.com", \\ - "subject":"package xyz", \\ - "account_id":"291f43c1-deae-431c-b763-712307fa6859", \\ - "body":"please install package xyz for use with Python3", \\ - "tags":["member"], \\ - "location":"/projects/0010abe1-9283-4b42-b403-fa4fc1e3be57/worksheet.sagews", \\ - "info":{"user_agent":"Mozilla/5.0","course":"cc8f1243-d573-4562-9aab-c15a3872d683"}}' \\ - https://cocalc.com/api/v1/create_support_ticket - ==> {"event":"support_ticket_url", - "id":"abd649bf-ea2d-4952-b925-e44c6903945e", - "url":"https://sagemathcloud.zendesk.com/requests/0123"} -\`\`\`\ -`, - }), -); - -message({ - // client ← hub - event: "support_ticket_url", - id: undefined, - url: required, -}); - -// client → hub -API( - message2({ - event: "get_support_tickets", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - }, - desc: `\ -Fetch information on support tickets for the user making the request. -See the example for details on what is returned. - -Notes: - -- There may be a delay of several minutes between the time a support ticket -is created with a given \`account_id\` and the time that ticket is -available to the account owner via \`get_support_tickets\`. -- Field \`account_id\` is not required because it is implicit from the request. -- Archived tickets are not returned. - -Example: - -\`\`\` -curl -u sk_abcdefQWERTY090900000000: -X POST \\ - https://cocalc.com/api/v1/get_support_tickets - ==> {"event":"support_tickets", - "id":"58bfd6f4-fd63-4602-82b8-676d92f8b0b8", - "tickets":[{"id":1234, - "subject":"package xyz", - "description":"package xyz\\n\\nhttps://cocalc.com/projects/0010abe1-9283-4b42-b403-fa4fc1e3be57/worksheet.sagews\\n\\nCourse: https://cocalc.com/projects/cc8f1243-d573-4562-9aab-c15a3872d683", - "created_at":"2017-07-05T14:28:38Z", - "updated_at":"2017-07-05T14:29:29Z", - "status":"open", - "url":"https://sagemathcloud.zendesk.com/requests/0123"}]} -\`\`\`\ -`, - }), -); - -message({ - // client ← hub - event: "support_tickets", - id: undefined, - tickets: required, -}); // json-list - /* Queries directly to the database (sort of like Facebook's GraphQL) */ From 197fe9a5d2d160ce73a2d28eb0e663a647f6e2b7 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 3 Feb 2025 15:36:45 +0000 Subject: [PATCH 106/281] nats: rewrite streaming LLM output to use ephemeral nats jetstreams and consumers --- src/packages/frontend/client/llm.ts | 39 +++++++++--------- src/packages/frontend/client/nats.ts | 31 +++++++++++++++ src/packages/hub/client.coffee | 22 ----------- src/packages/nats/hub-api/index.ts | 3 ++ src/packages/nats/hub-api/llm.ts | 12 ++++++ src/packages/pnpm-lock.yaml | 3 ++ src/packages/project/nats/api/index.ts | 4 +- src/packages/server/llm/index.ts | 49 ++++++++++++++--------- src/packages/server/llm/openai-lc.ts | 25 ++++++------ src/packages/server/nats/api/index.ts | 6 ++- src/packages/server/nats/api/llm.ts | 55 ++++++++++++++++++++++++++ src/packages/server/nats/auth.ts | 2 +- src/packages/server/package.json | 8 +++- src/packages/util/types/llm.ts | 7 +++- 14 files changed, 183 insertions(+), 83 deletions(-) create mode 100644 src/packages/nats/hub-api/llm.ts create mode 100644 src/packages/server/nats/api/llm.ts diff --git a/src/packages/frontend/client/llm.ts b/src/packages/frontend/client/llm.ts index b54f35b294..8ab9fbef4a 100644 --- a/src/packages/frontend/client/llm.ts +++ b/src/packages/frontend/client/llm.ts @@ -171,34 +171,31 @@ export class LLMClient { history = truncateHistory(history, maxTokens - n, model); } // console.log("chatgpt", { input, system, history, project_id, path }); - const mesg = message.chatgpt({ - text: input, + const options = { + input, system, project_id, path, history, model, tag: `app:${tag}`, - stream: chatStream != null, - }); + }; if (chatStream == null) { - return (await this.client.async_call({ message: mesg })).text; + // not streaming + return await this.client.nats_client.llm(options); } - chatStream.once("start", () => { + chatStream.once("start", async () => { // streaming version - this.client.call({ - message: mesg, - error_event: true, - cb: (err, resp) => { - if (err) { - chatStream.error(err); - } else { - chatStream.process(resp.text); - } - }, - }); + try { + await this.client.nats_client.llm({ + ...options, + stream: chatStream.process, + }); + } catch (err) { + chatStream.error(err); + } }); return "see stream for output"; @@ -343,14 +340,14 @@ class ChatStream extends EventEmitter { super(); } - process(text?: string) { + process = (text?: string) => { // emits undefined text when done (or err below) this.emit("token", text); - } + }; - error(err) { + error = (err) => { this.emit("error", err); - } + }; } export type { ChatStream }; diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index a4cd4a8587..df2ee8ec0f 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -16,6 +16,7 @@ import { getPrimusConnection } from "@cocalc/nats/primus"; import { isValidUUID } from "@cocalc/util/misc"; import { OpenFiles } from "@cocalc/nats/sync/open-files"; import { PubSub } from "@cocalc/nats/sync/pubsub"; +import type { ChatOptions } from "@cocalc/util/types/llm"; export class NatsClient { /*private*/ client: WebappClient; @@ -281,4 +282,34 @@ export class NatsClient { }) => { return new PubSub({ project_id, path, name, env: await this.getEnv() }); }; + + // Evaluate the llm. This streams the result if stream is given an option, + // AND it also always returns the result. + llm = async (opts: ChatOptions) => { + const { stream, ...options } = opts; + const { subject, streamName } = await this.hub.llm.evaluate(options); + // making an ephemeral consumer + const nc = await this.getConnection(); + const js = jetstream.jetstream(nc); + const jsm = await jetstream.jetstreamManager(nc); + const { name } = await jsm.consumers.add(streamName, { + filter_subject: subject, + }); + const consumer = await js.consumers.get(streamName, name); + const messages = await consumer.fetch(); + const decoder = new TextDecoder("utf-8"); + let accumulate = ""; + for await (const mesg of messages) { + if (mesg.data.length == 0) { + // done. + stream?.(undefined); // indicates done + messages.stop(); + break; + } + const text = decoder.decode(mesg.data); + accumulate += text; + stream?.(text); + } + return accumulate; + }; } diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee index cb7bf09942..0ef3a12db3 100644 --- a/src/packages/hub/client.coffee +++ b/src/packages/hub/client.coffee @@ -29,7 +29,6 @@ db_schema = require('@cocalc/util/db-schema') { REMEMBER_ME_COOKIE_NAME } = require("@cocalc/backend/auth/cookie-names"); generateHash = require("@cocalc/server/auth/hash").default; passwordHash = require("@cocalc/backend/auth/password-hash").default; -llm = require('@cocalc/server/llm/index'); jupyter_execute = require('@cocalc/server/jupyter/execute').execute; jupyter_kernels = require('@cocalc/server/jupyter/kernels').default; create_project = require("@cocalc/server/projects/create").default; @@ -1700,27 +1699,6 @@ class exports.Client extends EventEmitter dbg("failed -- #{err}") @error_to_client(id:mesg.id, error:"#{err}") - mesg_chatgpt: (mesg) => - dbg = @dbg("mesg_chatgpt") - dbg(mesg.text) - if not @account_id? - @error_to_client(id:mesg.id, error:"not signed in") - return - if mesg.stream - try - stream = (text) => - @push_to_client(message.chatgpt_response(id:mesg.id, text:text, multi_response:text?)) - await llm.evaluate(input:mesg.text, system:mesg.system, account_id:@account_id, project_id:mesg.project_id, path:mesg.path, history:mesg.history, model:mesg.model, tag:mesg.tag, stream:stream) - catch err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"#{err}") - else - try - output = await llm.evaluate(input:mesg.text, system:mesg.system, account_id:@account_id, project_id:mesg.project_id, path:mesg.path, history:mesg.history, model:mesg.model, tag:mesg.tag) - @push_to_client(message.chatgpt_response(id:mesg.id, text:output)) - catch err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"#{err}") # These are deprecated. Not the best approach. mesg_openai_embeddings_search: (mesg) => diff --git a/src/packages/nats/hub-api/index.ts b/src/packages/nats/hub-api/index.ts index d2c9e3ff56..4861155bbf 100644 --- a/src/packages/nats/hub-api/index.ts +++ b/src/packages/nats/hub-api/index.ts @@ -2,18 +2,21 @@ import { isValidUUID } from "@cocalc/util/misc"; import { type Purchases, purchases } from "./purchases"; import { type System, system } from "./system"; import { type DB, db } from "./db"; +import { type LLM, llm } from "./llm"; import { handleErrorMessage } from "@cocalc/nats/util"; export interface HubApi { system: System; db: DB; purchases: Purchases; + llm: LLM; } const HubApiStructure = { system, db, purchases, + llm, } as const; export function transformArgs({ name, args, account_id, project_id }) { diff --git a/src/packages/nats/hub-api/llm.ts b/src/packages/nats/hub-api/llm.ts new file mode 100644 index 0000000000..7b62a964ce --- /dev/null +++ b/src/packages/nats/hub-api/llm.ts @@ -0,0 +1,12 @@ +import { authFirst } from "./util"; +import type { ChatOptionsApi } from "@cocalc/util/types/llm"; + +export const llm = { + evaluate: authFirst, +}; + +export interface LLM { + evaluate: ( + opts: ChatOptionsApi, + ) => Promise<{ subject: string; streamName: string }>; +} diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 5ec69023ea..275ec041a6 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1474,6 +1474,9 @@ importers: '@langchain/openai': specifier: ^0.3.17 version: 0.3.17(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) + '@nats-io/jetstream': + specifier: 3.0.0-36 + version: 3.0.0-36 '@node-saml/passport-saml': specifier: ^4.0.4 version: 4.0.4 diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index de6ec3f64a..b4608e9c3b 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -62,13 +62,13 @@ async function listen(subscription, subject) { const { service } = request.args[0] ?? {}; if (service == "open-files") { terminateOpenFiles(); - mesg.respond(jc.encode({ status: "terminating", service })); + mesg.respond(jc.encode({ status: "terminated", service })); continue; } else if (service == "api") { // special hook so admin can terminate handling. This is useful for development. console.warn("TERMINATING listening on ", subject); logger.debug("TERMINATING listening on ", subject); - mesg.respond(jc.encode({ status: "terminating", service })); + mesg.respond(jc.encode({ status: "terminated", service })); subscription.unsubscribe(); return; } else { diff --git a/src/packages/server/llm/index.ts b/src/packages/server/llm/index.ts index 56f585aeaf..1d2748161f 100644 --- a/src/packages/server/llm/index.ts +++ b/src/packages/server/llm/index.ts @@ -42,7 +42,7 @@ import { model2vendor, } from "@cocalc/util/db-schema/llm-utils"; import { KUCALC_COCALC_COM } from "@cocalc/util/db-schema/site-defaults"; -import { ChatOptions, ChatOutput, History } from "@cocalc/util/types/llm"; +import type { ChatOptions, ChatOutput, History } from "@cocalc/util/types/llm"; import { checkForAbuse } from "./abuse"; import { evaluateAnthropic } from "./anthropic"; import { callChatGPTAPI } from "./call-llm"; @@ -97,15 +97,25 @@ function wrapStream(stream?: ChatOptions["stream"]) { const throttled = throttle( () => { - if (buffer.length === 0) return; - if (closed) throw new Error("stream closed"); + if (buffer.length === 0) { + return; + } + if (closed) { + throw new Error("stream closed"); + } // if the last object in buffer is the end object, remove it closed = buffer[buffer.length - 1] === end; - if (closed) buffer.pop(); + if (closed) { + buffer.pop(); + } const str = buffer.join(""); buffer.length = 0; - stream(str); - if (closed) stream(); + if (str.length > 0) { + stream(str); + } + if (closed) { + stream(); + } }, THROTTLE_STREAM_MS, { leading: true, trailing: true }, @@ -132,19 +142,20 @@ async function evaluateImpl({ stream, maxTokens, }: ChatOptions): Promise { - log.debug("evaluateImpl", { - input, - history, - system, - account_id, - analytics_cookie, - project_id, - path, - model, - tag, - stream: stream != null, - maxTokens, - }); + // LARGE -- e.g., complete input -- only uncomment when developing if you need this. + // log.debug("evaluateImpl", { + // input, + // history, + // system, + // account_id, + // analytics_cookie, + // project_id, + // path, + // model, + // tag, + // stream: stream != null, + // maxTokens, + // }); const start = Date.now(); await checkForAbuse({ account_id, analytics_cookie, model }); diff --git a/src/packages/server/llm/openai-lc.ts b/src/packages/server/llm/openai-lc.ts index 625e674dc3..360663783d 100644 --- a/src/packages/server/llm/openai-lc.ts +++ b/src/packages/server/llm/openai-lc.ts @@ -59,14 +59,15 @@ export async function evaluateOpenAILC( const isO1 = model != "o1-mini" && model != "o1"; const streaming = stream != null && isO1; - log.debug("evaluateOpenAILC", { - input, - history, - system, - model, - stream: streaming, - maxTokens, - }); + // This is also quite big -- only uncomment when developing and needing this. + // log.debug("evaluateOpenAILC", { + // input, + // history, + // system, + // model, + // stream: streaming, + // maxTokens, + // }); const params = mode === "cocalc" ? await getParams(model) : { apiKey: opts.apiKey, model }; @@ -93,9 +94,8 @@ export async function evaluateOpenAILC( inputMessagesKey: "input", historyMessagesKey: "history", getMessageHistory: async () => { - const { messageHistory, tokens } = await transformHistoryToMessages( - history, - ); + const { messageHistory, tokens } = + await transformHistoryToMessages(history); historyTokens = tokens; return messageHistory; }, @@ -127,7 +127,8 @@ export async function evaluateOpenAILC( output = content2string(content); } - log.debug("finalResult", finalResult); + // ATTENTION : Do *NOT* log this unless you are doing low level debugging. It could be pretty big... + // log.debug("finalResult", finalResult); // and an empty call when done opts.stream?.(); diff --git a/src/packages/server/nats/api/index.ts b/src/packages/server/nats/api/index.ts index 1e468fb47d..cc41a90746 100644 --- a/src/packages/server/nats/api/index.ts +++ b/src/packages/server/nats/api/index.ts @@ -67,13 +67,13 @@ export async function initAPI() { logger.debug(`Terminate service '${service}'`); if (service == "database") { terminateDatabase(); - mesg.respond(jc.encode({ status: "terminating", service })); + mesg.respond(jc.encode({ status: "terminated", service })); continue; } else if (service == "api") { // special hook so admin can terminate handling. This is useful for development. console.warn("TERMINATING listening on ", subject); logger.debug("TERMINATING listening on ", subject); - mesg.respond(jc.encode({ status: "terminating", service })); + mesg.respond(jc.encode({ status: "terminated", service })); sub.unsubscribe(); return; } else { @@ -104,11 +104,13 @@ async function handleApiRequest(request, mesg) { import * as purchases from "./purchases"; import * as db from "./db"; +import * as llm from "./llm"; import * as system from "./system"; export const hubApi: HubApi = { system, db, + llm, purchases, }; diff --git a/src/packages/server/nats/api/llm.ts b/src/packages/server/nats/api/llm.ts new file mode 100644 index 0000000000..1c28ec603d --- /dev/null +++ b/src/packages/server/nats/api/llm.ts @@ -0,0 +1,55 @@ +import { evaluate as evaluateStreaming } from "@cocalc/server/llm/index"; +import type { ChatOptionsApi } from "@cocalc/util/types/llm"; +import { jetstreamManager, type StoreCompression } from "@nats-io/jetstream"; +import { getConnection } from "@cocalc/backend/nats"; +import { isValidUUID } from "@cocalc/util/misc"; +import { v4 } from "uuid"; + +const ONE_MINUTE_IN_NANOS = 1000 * 1000 * 1000 * 60; + +export async function evaluate( + options: ChatOptionsApi, +): Promise<{ subject: string; streamName: string }> { + if (!options.system?.trim()) { + // I noticed in testing that for some models they just fail, so let's be clear immediately. + throw Error("the system prompt MUST be nonempty"); + } + if (!isValidUUID(options.account_id)) { + throw Error("account_id must be a valid uuid"); + } + + const id = v4().slice(0, 8); + const streamName = `llm-account-${options.account_id}`; + const subject = `llm.account-${options.account_id}.${id}`; + const nc = await getConnection(); + const jsm = await jetstreamManager(nc); + const streamOptions = { + subjects: [subject], + compression: "s2" as StoreCompression, + // max_age: browser has 5 minutes (in nanoseconds) to get their messages from the stream + max_age: 5 * ONE_MINUTE_IN_NANOS, + }; + try { + await jsm.streams.add({ ...streamOptions, name: streamName }); + } catch { + await jsm.streams.update(streamName, streamOptions); + } + + const stream = (text: string) => { + nc.publish(subject, text); + }; + + const f = async () => { + try { + await evaluateStreaming({ + ...options, + stream, + }); + } catch (err) { + stream(`${err}`); + } + }; + f(); + + return { subject, streamName }; +} diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index e5051eb5ba..9955f52405 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -119,7 +119,7 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { const goalSub = new Set(["_INBOX.>", "$JS.API.>"]); if (userType == "account") { - goalSub.add(`$KV.account-${userId}.>`); + goalSub.add(`*.account-${userId}.>`); const pool = getPool(); // all RUNNING projects with the user's group diff --git a/src/packages/server/package.json b/src/packages/server/package.json index 952272fa39..567fd12afb 100644 --- a/src/packages/server/package.json +++ b/src/packages/server/package.json @@ -24,7 +24,10 @@ "./settings": "./dist/settings/index.js", "./settings/*": "./dist/settings/*.js" }, - "keywords": ["utilities", "cocalc"], + "keywords": [ + "utilities", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf node_modules dist", @@ -38,8 +41,8 @@ "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/database": "workspace:*", - "@cocalc/nats": "workspace:*", "@cocalc/gcloud-pricing-calculator": "^1.14.0", + "@cocalc/nats": "workspace:*", "@cocalc/server": "workspace:*", "@cocalc/util": "workspace:*", "@google-ai/generativelanguage": "^2.7.0", @@ -56,6 +59,7 @@ "@langchain/google-genai": "^0.1.6", "@langchain/mistralai": "^0.2.0", "@langchain/openai": "^0.3.17", + "@nats-io/jetstream": "3.0.0-36", "@node-saml/passport-saml": "^4.0.4", "@passport-js/passport-twitter": "^1.0.8", "@passport-next/passport-google-oauth2": "^1.0.0", diff --git a/src/packages/util/types/llm.ts b/src/packages/util/types/llm.ts index 31db64d2ce..b8b64404cb 100644 --- a/src/packages/util/types/llm.ts +++ b/src/packages/util/types/llm.ts @@ -12,7 +12,7 @@ export interface ChatOutput { completion_tokens: number; } -export interface ChatOptions { +export interface ChatOptionsApi { input: string; // new input that user types system?: string; // extra setup that we add for relevance and context account_id?: string; @@ -22,13 +22,16 @@ export interface ChatOptions { history?: History; model?: LanguageModel; // default is defined by server setting default_llm tag?: string; + maxTokens?: number; +} + +export interface ChatOptions extends ChatOptionsApi { // If stream is set, then everything works as normal with two exceptions: // - The stream function is called with bits of the output as they are produced, // until the output is done and then it is called with undefined. // - Maybe the total_tokens, which is stored in the database for analytics, // might be off: https://community.openai.com/t/openai-api-get-usage-tokens-in-response-when-set-stream-true/141866 stream?: (output?: string) => void; - maxTokens?: number; } // This could be Ollama or CustomOpenAI From 5875d373089a5a54c614a1faf6911dea02214832 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 00:36:05 +0000 Subject: [PATCH 107/281] nats: implement a very cool key:value store wrapper, which we're going to use for public system configuration info, i.e., rewrite /customize, etc. --- src/packages/frontend/client/nats.ts | 11 ++ src/packages/nats/primus.ts | 2 +- src/packages/nats/sync/kv.ts | 150 ++++++++++++++++++ src/packages/nats/sync/open-files.ts | 5 +- src/packages/nats/sync/pubsub.ts | 2 +- src/packages/nats/sync/synctable-kv-atomic.ts | 37 ++++- src/packages/nats/sync/synctable-kv.ts | 13 +- src/packages/nats/sync/synctable-stream.ts | 8 +- src/packages/nats/sync/synctable.ts | 3 +- src/packages/nats/system.ts | 28 ++++ src/packages/nats/types.ts | 6 + src/packages/nats/util.ts | 13 +- src/packages/server/nats/auth.ts | 12 +- 13 files changed, 259 insertions(+), 31 deletions(-) create mode 100644 src/packages/nats/sync/kv.ts create mode 100644 src/packages/nats/system.ts create mode 100644 src/packages/nats/types.ts diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index df2ee8ec0f..2ca5043343 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -17,6 +17,7 @@ import { isValidUUID } from "@cocalc/util/misc"; import { OpenFiles } from "@cocalc/nats/sync/open-files"; import { PubSub } from "@cocalc/nats/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; +import { SystemKv } from "@cocalc/nats/system"; export class NatsClient { /*private*/ client: WebappClient; @@ -28,6 +29,7 @@ export class NatsClient { public hub: HubApi; public sessionId = randomId(); private openFilesCache: { [project_id: string]: OpenFiles } = {}; + private theSystemKv?: SystemKv; constructor(client: WebappClient) { this.client = client; @@ -312,4 +314,13 @@ export class NatsClient { } return accumulate; }; + + systemKv = reuseInFlight(async () => { + if (this.theSystemKv == null) { + const s = new SystemKv(await this.getEnv()); + await s.init(); + this.theSystemKv = s; + } + return this.theSystemKv!; + }); } diff --git a/src/packages/nats/primus.ts b/src/packages/nats/primus.ts index 048521010f..abc786c2c9 100644 --- a/src/packages/nats/primus.ts +++ b/src/packages/nats/primus.ts @@ -38,7 +38,7 @@ s9.sparks['cb'].write('blah') */ import { EventEmitter } from "events"; -import { type NatsEnv } from "@cocalc/nats/sync/synctable-kv"; +import { type NatsEnv } from "@cocalc/nats/types"; import { delay } from "awaiting"; export type Role = "client" | "server"; diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts new file mode 100644 index 0000000000..d85016b4d6 --- /dev/null +++ b/src/packages/nats/sync/kv.ts @@ -0,0 +1,150 @@ +/* +This is a simple KV wrapper around NATS's KV, for small KV stores, suitable for configuration data. + +- it emits an event ('change', key, value) whenever anything changes + +- explicitly call "await this.init()" to initialize it + +- calling get() synchronously provides ALL the data. + +- call await set({key:value, key2:value2, ...}) to set data, with the following semantics: + + - set ONLY makes a change if our local version (this.get()[key]) of the value is different from + what you're trying to set the value to, where different is defiend by lodash isEqual. + + - if our local version this.get()[key] was not the most recent version in NATS, then the set will + definitely throw an exception! This is fantastic because it means you can modify and save what + is in the local cache on multiple nodes at once anywhere, and be 100% certain to never overwrite + data in complicated objects. Of course, you have to assume "await set()" will sometimes fail. + + - set "pipelines" in that MAX_PARALLEL_SET key/value pairs are set at once, without waiting + for each set to get ACK'd from the server before doing more sets. This makes this massively + faster for bigger objects, but means that if "await set({...})" fails, you don't immediately + know which keys were successfully set and which failed, though all that worked will get + updated soon and reflected in get(). +*/ + +import { EventEmitter } from "events"; +import { type NatsEnv } from "@cocalc/nats/types"; +import { Kvm } from "@nats-io/kv"; +import { getAllFromKv } from "@cocalc/nats/util"; +import { isEqual } from "lodash"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { map as awaitMap } from "awaiting"; + +const MAX_PARALLEL_SET = 50; + +export class KV extends EventEmitter { + public readonly name: string; + private options?; + private env: NatsEnv; + private kv?; + private watch?; + private all?: { [key: string]: any }; + private revisions?: { [key: string]: number }; + + constructor({ + name, + env, + options, + }: { + name: string; + env: NatsEnv; + options?; + }) { + super(); + this.env = env; + this.name = name; + this.options = options; + } + + init = reuseInFlight(async () => { + if (this.all != null) { + return; + } + const kvm = new Kvm(this.env.nc); + this.kv = await kvm.create(this.name, { + compression: true, + ...this.options, + }); + const { all, revisions } = await getAllFromKv({ + kv: this.kv, + }); + this.revisions = revisions; + for (const k in all) { + all[k] = this.env.jc.decode(all[k]); + } + this.all = all; + this.emit("connected"); + this.startWatch(); + }); + + private startWatch = async () => { + // watch for changes + this.watch = await this.kv.watch({ + // we assume that we ONLY delete old items which are not relevant + ignoreDeletes: true, + include: "updates", + }); + //for await (const { key, value } of this.watch) { + for await (const { revision, key, value } of this.watch) { + if (this.revisions == null || this.all == null) { + return; + } + this.revisions[key] = revision; + if (value.length == 0) { + // delete + delete this.all[key]; + } else { + this.all[key] = this.env.jc.decode(value); + } + this.emit("change", key, this.all[key]); + } + }; + + close = () => { + this.watch?.stop(); + delete this.all; + delete this.revisions; + this.emit("closed"); + this.removeAllListeners(); + }; + + get = () => { + return { ...this.all }; + }; + + delete = async (key) => { + if (this.all == null) { + throw Error("not ready"); + } + if (this.all[key] != null) { + await this.kv.delete(key); + } + delete this.all[key]; + }; + + set = async (obj) => { + await awaitMap( + Object.keys(obj), + MAX_PARALLEL_SET, + async (key) => await this.setOne(key, obj[key]), + ); + }; + + private setOne = async (key, value) => { + if (this.all == null || this.revisions == null) { + throw Error("not ready"); + } + if (isEqual(this.all[key], value)) { + return; + } + const revision = this.revisions[key]; + const val = this.env.jc.encode(value); + const newRevision = await this.kv.put(key, val, { + previousSeq: revision, + }); + this.revisions[key] = newRevision; + this.all[key] = val; + }; +} diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 70aff29997..2d7963ed41 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -28,7 +28,8 @@ null */ -import { getKv, type NatsEnv } from "./synctable-kv"; +import { type NatsEnv } from "@cocalc/nats/types"; +import { getKv } from "./synctable-kv"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { sha1 } from "@cocalc/util/misc"; import { isEqual } from "lodash"; @@ -92,8 +93,8 @@ export class OpenFiles { this.state = "closed"; for (const w of this.watches) { w.stop(); - this.watches = []; } + this.watches = []; }; // When a client has a file open, they should periodically diff --git a/src/packages/nats/sync/pubsub.ts b/src/packages/nats/sync/pubsub.ts index 7b782c8508..d839f3e39f 100644 --- a/src/packages/nats/sync/pubsub.ts +++ b/src/packages/nats/sync/pubsub.ts @@ -3,7 +3,7 @@ Use NATS simple pub/sub to share state for something *ephemeral* in a project. */ import { projectSubject } from "@cocalc/nats/names"; -import { type NatsEnv } from "./synctable-kv"; +import { type NatsEnv } from "@cocalc/nats/types"; import { EventEmitter } from "events"; import { State } from "./synctable-kv-atomic"; diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index 3adc576adc..add0da192d 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -1,6 +1,32 @@ +/* + + +TODO: This is a kv store where you atomically do updates. The way this is written, two clents +might make a change to the same object at the same time and one overwrites the other. +However, I just realized with NATS we can easily prevent this!! There is a version +option to update, so using that instead of put make it possible to detect if there's a +potential conflict, then fix and retry!!! + +(See packages/nats/sync/kv.ts for how to do this properly!) + + * Updates the existing entry provided that the previous sequence + * for the Kv is at the specified version. This ensures that the + * KV has not been modified prior to the update. + * @param k + * @param data + * @param version + update(k: string, data: Payload, version: number): Promise; + + +The synctable-kv.ts file has another one where each key:value in a single object is its own key:value +in the store. + +*/ + import { keys } from "lodash"; import { client_db } from "@cocalc/util/db-schema/client-db"; -import { getKv, toKey, type NatsEnv, natsKeyPrefix } from "./synctable-kv"; +import { getKv, toKey, natsKeyPrefix } from "./synctable-kv"; +import { type NatsEnv } from "@cocalc/nats/types"; import { sha1 } from "@cocalc/util/misc"; import { EventEmitter } from "events"; import { getAllFromKv } from "@cocalc/nats/util"; @@ -17,6 +43,7 @@ export class SyncTableKVAtomic extends EventEmitter { private project_id?: string; private account_id?: string; private state: State = "disconnected"; + private watches: any[] = []; constructor({ query, @@ -95,7 +122,7 @@ export class SyncTableKVAtomic extends EventEmitter { get = async (obj?, options: { natsKeys?: boolean } = {}) => { if (obj == null) { - const raw = await getAllFromKv({ + const { all: raw } = await getAllFromKv({ kv: this.kv, key: `${this.natsKeyPrefix}.>`, }); @@ -124,6 +151,7 @@ export class SyncTableKVAtomic extends EventEmitter { key: `${this.natsKeyPrefix}.>`, include: "updates", }); + this.watches.push(w); for await (const { value } of w) { if (this.state == "closed") { return; @@ -135,6 +163,9 @@ export class SyncTableKVAtomic extends EventEmitter { close = () => { this.set_state("closed"); this.removeAllListeners(); - // TODO: stop watchers... ? + for (const w of this.watches) { + w.stop(); + } + this.watches = []; }; } diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 8c46e347fc..f134ac3bf3 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -19,6 +19,7 @@ import { wait } from "@cocalc/util/async-wait"; import { throttle } from "lodash"; import { fromJS, Map } from "immutable"; import { getAllFromKv } from "@cocalc/nats/util"; +import { type NatsEnv } from "@cocalc/nats/types"; export function natsKeyPrefix({ query, @@ -68,13 +69,6 @@ export async function getKv({ return await kvm.create(name, { compression: true, ...options }); } -export interface NatsEnv { - nc; // nats connection - jc; // jsoncodec - // compute sha1 hash efficiently (set differently on backend) - sha1?: (string) => string; -} - export function toKey(x): string | undefined { if (x === undefined) { return undefined; @@ -380,7 +374,10 @@ export class SyncTableKV extends EventEmitter { const kv = await this.getKv(); if (obj == null) { // everything known in this table by the project - const raw = await getAllFromKv({ kv, key: `${this.natsKeyPrefix}.>` }); + const { all: raw } = await getAllFromKv({ + kv, + key: `${this.natsKeyPrefix}.>`, + }); const all: any = {}; for (const key in raw) { const x = raw[key]; diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts index bb44b221ce..eb36fa447f 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/nats/sync/synctable-stream.ts @@ -15,16 +15,10 @@ import { keys } from "lodash"; import { cmp_Date, is_array, isValidUUID, sha1 } from "@cocalc/util/misc"; import { client_db } from "@cocalc/util/db-schema/client-db"; import { EventEmitter } from "events"; +import { type NatsEnv } from "@cocalc/nats/types"; export type State = "disconnected" | "connected" | "closed"; -interface NatsEnv { - nc; // nats connection - jc; // jsoncodec - // compute sha1 hash efficiently (set differently on backend) - sha1?: (string) => string; -} - function toKey(x): string | undefined { if (x === undefined) { return undefined; diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/nats/sync/synctable.ts index 7743db7bc0..580735743e 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/nats/sync/synctable.ts @@ -1,4 +1,5 @@ -import { SyncTableKV, type NatsEnv } from "./synctable-kv"; +import { type NatsEnv } from "@cocalc/nats/types"; +import { SyncTableKV } from "./synctable-kv"; import { SyncTableKVAtomic } from "./synctable-kv-atomic"; import { SyncTableStream } from "./synctable-stream"; diff --git a/src/packages/nats/system.ts b/src/packages/nats/system.ts new file mode 100644 index 0000000000..c10a119d83 --- /dev/null +++ b/src/packages/nats/system.ts @@ -0,0 +1,28 @@ +/* +This is a key:value store that hubs can write to and all +users of cocalc can read from. It contains: + +- recent system-wide notifications that haven't been canceled + system.notifications.{random} + +- the customize data: what used to be the /customize http endpoint + this makes it so clients get notified whenever anything changes, e.g., when the + recommended or required version changes, and can act accordingly. The UI + can also change. + +Development: + +~/cocalc/src/packages/server$ n +Welcome to Node.js v18.17.1. +Type ".help" for more information. +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/system"); s = new a.SystemKv(env); await s.init(); + +*/ + +import { KV } from "@cocalc/nats/sync/kv"; + +export class SystemKv extends KV { + constructor(env) { + super({ env, name: "system" }); + } +} diff --git a/src/packages/nats/types.ts b/src/packages/nats/types.ts new file mode 100644 index 0000000000..4bafa1cb2a --- /dev/null +++ b/src/packages/nats/types.ts @@ -0,0 +1,6 @@ +export interface NatsEnv { + nc; // nats connection + jc; // jsoncodec + // compute sha1 hash efficiently (set differently on backend) + sha1?: (string) => string; +} diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index 1cdd46c62d..5d8d18d319 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -24,17 +24,22 @@ export async function getAllFromKv({ kv; key?: string | string[]; timeout?: number; -}): Promise<{ [key: string]: any }> { +}): Promise<{ + all: { [key: string]: any }; + revisions: { [key: string]: number }; +}> { const total = await numKeys(kv, key); let count = 0; const all: any = {}; + const revisions: { [key: string]: number } = {}; if (total == 0) { - return all; + return { all, revisions }; } const watch = await kv.watch({ key, ignoreDeletes: true }); let id: any = 0; - for await (const { key, value } of watch) { + for await (const { key, value, revision } of watch) { all[key] = value; + revisions[key] = revision; count += 1; @@ -55,7 +60,7 @@ export async function getAllFromKv({ watch.stop(); }, timeout); } - return all; + return { all, revisions }; } export function handleErrorMessage(mesg) { diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 9955f52405..b9dd410b7a 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -112,11 +112,15 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { const userType = getCoCalcUserType(cocalcUser); // TODO: jetstream permissions are WAY TO BROAD. const goalPub = new Set([ - "_INBOX.>", - `hub.${userType}.${userId}.>`, - "$JS.API.>", + "_INBOX.>", // so can use request/response + `hub.${userType}.${userId}.>`, // can talk as *only this user* to the hub's api's + "$JS.API.>", // so can use Jestream: TODO: too much???! + ]); + const goalSub = new Set([ + "_INBOX.>", // so can user request/response + "$JS.API.>", // TODO! This needs to be restrained more, I think??! Don't know. + "system.>", // access to READ the system info kv store. ]); - const goalSub = new Set(["_INBOX.>", "$JS.API.>"]); if (userType == "account") { goalSub.add(`*.account-${userId}.>`); From c5af66304e780cad7dd2e1c05914210404bbcca4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 02:29:44 +0000 Subject: [PATCH 108/281] nats: framework for system config -- not used yet --- .../nats/{index.ts => changefeeds.ts} | 0 src/packages/nats/sync/kv.ts | 8 +++++-- src/packages/server/nats/api/index.ts | 2 +- src/packages/server/nats/index.ts | 2 +- src/packages/server/nats/system.ts | 22 +++++++++++++++++++ src/packages/sync/table/synctable.ts | 2 +- 6 files changed, 31 insertions(+), 5 deletions(-) rename src/packages/database/nats/{index.ts => changefeeds.ts} (100%) create mode 100644 src/packages/server/nats/system.ts diff --git a/src/packages/database/nats/index.ts b/src/packages/database/nats/changefeeds.ts similarity index 100% rename from src/packages/database/nats/index.ts rename to src/packages/database/nats/changefeeds.ts diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index d85016b4d6..6b8a1ada5d 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -115,11 +115,12 @@ export class KV extends EventEmitter { }; delete = async (key) => { - if (this.all == null) { + if (this.all == null || this.revisions == null) { throw Error("not ready"); } if (this.all[key] != null) { - await this.kv.delete(key); + const newRevision = await this.kv.delete(key); + this.revisions[key] = newRevision; } delete this.all[key]; }; @@ -139,6 +140,9 @@ export class KV extends EventEmitter { if (isEqual(this.all[key], value)) { return; } + if (value === undefined) { + return await this.delete(key); + } const revision = this.revisions[key]; const val = this.env.jc.encode(value); const newRevision = await this.kv.put(key, val, { diff --git a/src/packages/server/nats/api/index.ts b/src/packages/server/nats/api/index.ts index cc41a90746..572ce5e26e 100644 --- a/src/packages/server/nats/api/index.ts +++ b/src/packages/server/nats/api/index.ts @@ -39,7 +39,7 @@ import getLogger from "@cocalc/backend/logger"; import { type HubApi, getUserId, transformArgs } from "@cocalc/nats/hub-api"; import { getConnection } from "@cocalc/backend/nats"; import userIsInGroup from "@cocalc/server/accounts/is-in-group"; -import { terminate as terminateDatabase } from "@cocalc/database/nats"; +import { terminate as terminateDatabase } from "@cocalc/database/nats/changefeeds"; const logger = getLogger("server:nats:api"); diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts index 4057d725d8..cffb42b33e 100644 --- a/src/packages/server/nats/index.ts +++ b/src/packages/server/nats/index.ts @@ -1,6 +1,6 @@ import getLogger from "@cocalc/backend/logger"; import { initAPI } from "./api"; -import { init as initDatabase } from "@cocalc/database/nats"; +import { init as initDatabase } from "@cocalc/database/nats/changefeeds"; const logger = getLogger("server:nats"); diff --git a/src/packages/server/nats/system.ts b/src/packages/server/nats/system.ts new file mode 100644 index 0000000000..d23fdcb70f --- /dev/null +++ b/src/packages/server/nats/system.ts @@ -0,0 +1,22 @@ +/* +This seems like it will be really useful... but we're not +using it yet. +*/ + +import { SystemKv } from "@cocalc/nats/system"; +import { JSONCodec } from "nats"; +import { getConnection } from "@cocalc/backend/nats"; +import { sha1 } from "@cocalc/backend/misc_node"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; + +let cache: SystemKv | null = null; +export const systemKv = reuseInFlight(async () => { + if (cache != null) { + return cache; + } + const jc = JSONCodec(); + const nc = await getConnection(); + cache = new SystemKv({ jc, nc, sha1 }); + await cache.init(); + return cache; +}); diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index baa87a6490..4cee800248 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -18,7 +18,7 @@ ways of orchestrating a SyncTable. let DEBUG: boolean = false; // enable experimental nats database backed changefeed. -// for this to work you must explicitly run the server in @cocalc/database/nats +// for this to work you must explicitly run the server in @cocalc/database/nats/changefeeds const USE_NATS = true; export function set_debug(x: boolean): void { From f9237d3fa8e7287209b42a13cced5a892d5cf0da Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 13:12:32 +0000 Subject: [PATCH 109/281] nats hub: add authtoken rpc --- src/packages/database/package.json | 7 +-- .../database/postgres-server-queries.coffee | 50 ------------------ src/packages/nats/hub-api/system.ts | 10 ++++ src/packages/nats/package.json | 12 +---- src/packages/server/accounts/is-admin.ts | 5 ++ src/packages/server/accounts/is-in-group.ts | 2 +- src/packages/server/auth/auth-token.ts | 52 +++++++++++++++++++ src/packages/server/nats/api/system.ts | 5 ++ 8 files changed, 77 insertions(+), 66 deletions(-) create mode 100644 src/packages/server/accounts/is-admin.ts create mode 100644 src/packages/server/auth/auth-token.ts diff --git a/src/packages/database/package.json b/src/packages/database/package.json index 1e5c4f60a1..6e1443c98b 100644 --- a/src/packages/database/package.json +++ b/src/packages/database/package.json @@ -5,7 +5,7 @@ "exports": { ".": "./dist/index.js", "./accounts/*": "./dist/accounts/*.js", - "./nats": "./dist/nats/index.js", + "./nats/*": "./dist/nats/*.js", "./pool": "./dist/pool/index.js", "./pool/*": "./dist/pool/*.js", "./postgres/*": "./dist/postgres/*.js", @@ -59,10 +59,7 @@ "url": "https://github.com/sagemathinc/cocalc" }, "homepage": "https://github.com/sagemathinc/cocalc", - "keywords": [ - "postgresql", - "cocalc" - ], + "keywords": ["postgresql", "cocalc"], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "bugs": { diff --git a/src/packages/database/postgres-server-queries.coffee b/src/packages/database/postgres-server-queries.coffee index 96c826c8e7..f8a66f64ca 100644 --- a/src/packages/database/postgres-server-queries.coffee +++ b/src/packages/database/postgres-server-queries.coffee @@ -1168,56 +1168,6 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext opts.cb(err) ) - ### - User auth token - ### - # save an auth token in the database - save_auth_token: (opts) => - opts = defaults opts, - account_id : required - auth_token : required - ttl : 12*3600 # ttl in seconds (default: 12 hours) - cb : required - if not @_validate_opts(opts) then return - @_query - query : 'INSERT INTO auth_tokens' - values : - 'auth_token :: CHAR(24) ' : opts.auth_token - 'expire :: TIMESTAMP ' : expire_time(opts.ttl) - 'account_id :: UUID ' : opts.account_id - cb : opts.cb - - # Get account_id of account with given auth_token. If it - # is not defined, get back undefined instead. - get_auth_token_account_id: (opts) => - opts = defaults opts, - auth_token : required - cb : required # cb(err, account_id) - @_query - query : 'SELECT account_id, expire FROM auth_tokens' - where : - 'auth_token = $::CHAR(24)' : opts.auth_token - cb : one_result (err, x) => - if err - opts.cb(err) - else if not x? - opts.cb() # nothing - else if x.expire <= new Date() - opts.cb() - else - opts.cb(undefined, x.account_id) - - delete_auth_token: (opts) => - opts = defaults opts, - auth_token : required - cb : undefined # cb(err) - @_query - query : 'DELETE FROM auth_tokens' - where : - 'auth_token = $::CHAR(24)' : opts.auth_token - cb : opts.cb - - ### Password reset ### diff --git a/src/packages/nats/hub-api/system.ts b/src/packages/nats/hub-api/system.ts index a38ad79fa5..e96d3c1c35 100644 --- a/src/packages/nats/hub-api/system.ts +++ b/src/packages/nats/hub-api/system.ts @@ -12,6 +12,8 @@ export const system = { terminate: authFirst, userTracking: authFirst, manageApiKeys: authFirst, + generateUserAuthToken: authFirst, + revokeUserAuthToken: noAuth, }; export interface System { @@ -40,4 +42,12 @@ export interface System { expire?: Date; id?: number; }) => Promise; + + generateUserAuthToken: (opts: { + account_id?: string; + user_account_id: string; + password?: string; + }) => Promise; + + revokeUserAuthToken: (authToken: string) => Promise; } diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 1b2d46bad7..6eafc92ba0 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -16,17 +16,9 @@ "test": "pnpm exec jest", "prepublishOnly": "pnpm test" }, - "files": [ - "dist/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "nats", - "cocalc" - ], + "keywords": ["utilities", "nats", "cocalc"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/comm": "workspace:*", diff --git a/src/packages/server/accounts/is-admin.ts b/src/packages/server/accounts/is-admin.ts new file mode 100644 index 0000000000..3524417dc0 --- /dev/null +++ b/src/packages/server/accounts/is-admin.ts @@ -0,0 +1,5 @@ +import userIsInGroup from "./is-in-group"; + +export default async function isAdmin(account_id: string): Promise { + return await userIsInGroup(account_id, "admin"); +} diff --git a/src/packages/server/accounts/is-in-group.ts b/src/packages/server/accounts/is-in-group.ts index 01e4b5a86e..a8fad78896 100644 --- a/src/packages/server/accounts/is-in-group.ts +++ b/src/packages/server/accounts/is-in-group.ts @@ -5,7 +5,7 @@ export type Group = "admin" | "partner" | "crm"; export default async function userIsInGroup( account_id: string, group: Group, -): Promise { +): Promise { const pool = getPool("long"); const { rows } = await pool.query( "SELECT groups FROM accounts WHERE account_id=$1", diff --git a/src/packages/server/auth/auth-token.ts b/src/packages/server/auth/auth-token.ts new file mode 100644 index 0000000000..917c5a3021 --- /dev/null +++ b/src/packages/server/auth/auth-token.ts @@ -0,0 +1,52 @@ +import isAdmin from "@cocalc/server/accounts/is-admin"; +import { generate } from "random-key"; +import getPool from "@cocalc/database/pool"; +import isPasswordCorrect from "./is-password-correct"; + +// map {account_id:{user_account_id:timestamp}} +const ban: { [account_id: string]: { [user_account_id: string]: number } } = {}; +const BAN_TIME_MS = 1000 * 30; + +export async function generateUserAuthToken({ + account_id, + user_account_id, + password = "", +}: { + account_id?: string; + user_account_id: string; + password?: string; +}): Promise { + if (account_id == null) { + throw Error("must specify account_id"); + } + const b = ban[account_id]?.[user_account_id]; + if (b && Date.now() - b < BAN_TIME_MS) { + throw Error( + `banned -- please wait at least #{BAN_TIME_MS/1000}s before trying again`, + ); + } + if (!(await isAdmin(account_id))) { + // not admin, so check password + if (!(await isPasswordCorrect({ account_id: user_account_id, password }))) { + if (ban[account_id] == null) { + ban[account_id] = {}; + } + ban[account_id][user_account_id] = Date.now(); + throw Error("incorrect password"); + } + } + + // ready to go + const authToken = generate(24); + const pool = getPool(); + await pool.query( + "INSERT INTO auth_tokens (auth_token, expire, account_id) VALUES($1, NOW()+INTERVAL '12 hours', $2)", + [authToken, account_id], + ); + return authToken; +} + +export async function revokeUserAuthToken(authToken: string) { + const pool = getPool(); + await pool.query("DELETE FROM auth_tokens WHERE auth_token=$1", [authToken]); +} diff --git a/src/packages/server/nats/api/system.ts b/src/packages/server/nats/api/system.ts index 4721021135..9719e05fa8 100644 --- a/src/packages/server/nats/api/system.ts +++ b/src/packages/server/nats/api/system.ts @@ -24,3 +24,8 @@ export async function userTracking({ }): Promise { await record_user_tracking(db(), account_id!, event, value); } + +export { + generateUserAuthToken, + revokeUserAuthToken, +} from "@cocalc/server/auth/auth-token"; From bb106ff4208b3b313e2555a4bf90dfc177396881 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 13:42:52 +0000 Subject: [PATCH 110/281] database: rewrite central log save in typescript --- .../database/postgres-server-queries.coffee | 39 +++-------------- src/packages/database/postgres/central-log.ts | 43 +++++++++++++++++++ src/packages/database/postgres/pii.ts | 8 ++-- .../database/postgres/server-settings.ts | 8 ++-- src/packages/hub/analytics.ts | 2 +- src/packages/hub/servers/robots.ts | 3 +- src/packages/server/auth/auth-token.ts | 8 +++- src/packages/server/mentions/handle.ts | 3 +- 8 files changed, 66 insertions(+), 48 deletions(-) create mode 100644 src/packages/database/postgres/central-log.ts diff --git a/src/packages/database/postgres-server-queries.coffee b/src/packages/database/postgres-server-queries.coffee index f8a66f64ca..e995ae09e7 100644 --- a/src/packages/database/postgres-server-queries.coffee +++ b/src/packages/database/postgres-server-queries.coffee @@ -62,19 +62,10 @@ read = require('read') passwordHash = require("@cocalc/backend/auth/password-hash").default; registrationTokens = require('./postgres/registration-tokens').default; {updateUnreadMessageCount} = require('./postgres/messages'); +centralLog = require('./postgres/central-log').default; stripe_name = require('@cocalc/util/stripe/name').default; -# log events, which contain personal information (email, account_id, ...) -PII_EVENTS = ['create_account', - 'change_password', - 'change_email_address', - 'webapp-add_passport', - 'get_user_auth_token', - 'successful_sign_in', - 'webapp-email_sign_up', - 'create_account_registration_token' - ] exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext # write an event to the central_log table @@ -83,27 +74,11 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext event : required # string value : required # object cb : undefined - - # always expire central_log entries after 1 year, unless … - expire = expire_time(365*24*60*60) - # exception events expire earlier - if opts.event == 'uncaught_exception' - expire = misc.expire_time(30 * 24 * 60 * 60) # del in 30 days - else - # and user-related events according to the PII time, although "never" falls back to 1 year - v = opts.value - if v.ip_address? or v.email_address? or opts.event in PII_EVENTS - expire = await pii_expire(@) ? expire - - @_query - query : 'INSERT INTO central_log' - values : - 'id::UUID' : misc.uuid() - 'event::TEXT' : opts.event - 'value::JSONB' : opts.value - 'time::TIMESTAMP' : 'NOW()' - 'expire::TIMESTAMP' : expire - cb : (err) => opts.cb?(err) + try + await centralLog(opts) + opts.cb?() + catch err + opts.cb?(err) uncaught_exception: (err) => # call when things go to hell in some unexpected way; at least @@ -1255,7 +1230,7 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext return # If expire no pii expiration is set, use 1 year as a fallback - expire = await pii_expire(@) ? expire_time(365*24*60*60) + expire = await pii_expire() ? expire_time(365*24*60*60) @_query query : 'INSERT INTO file_access_log' diff --git a/src/packages/database/postgres/central-log.ts b/src/packages/database/postgres/central-log.ts new file mode 100644 index 0000000000..ec14314308 --- /dev/null +++ b/src/packages/database/postgres/central-log.ts @@ -0,0 +1,43 @@ +import getPool from "@cocalc/database/pool"; +import { pii_expire } from "./pii"; +import { uuid } from "@cocalc/util/misc"; + +// log events, which contain personal information (email, account_id, ...) +const PII_EVENTS = new Set([ + "create_account", + "change_password", + "change_email_address", + "webapp-add_passport", + "get_user_auth_token", + "successful_sign_in", + "webapp-email_sign_up", + "create_account_registration_token", +]); + +export default async function centralLog({ + event, + value, +}: { + event: string; + value: object; +}) { + const pool = getPool(); + + let expire; + if (value["ip_address"] || value["email_address"] || PII_EVENTS.has(event)) { + const date = await pii_expire(); + if (date == null) { + expire = "NOW() + INTERVAL '6 MONTHS'"; + } else { + expire = `NOW() + INTERVAL '${(date.valueOf() - Date.now()) / 1000} seconds'`; + } + } else if (event == "uncaught_exception") { + expire = "NOW() + INTERVAL '1 MONTH'"; + } else { + expire = "NOW() + INTERVAL '1 YEAR'"; + } + await pool.query( + `INSERT INTO central_log(id,event,value,time,expire) VALUES($1,$2,$3,NOW(),${expire})`, + [uuid(), event, value], + ); +} diff --git a/src/packages/database/postgres/pii.ts b/src/packages/database/postgres/pii.ts index c260b73fde..7e09835d7a 100644 --- a/src/packages/database/postgres/pii.ts +++ b/src/packages/database/postgres/pii.ts @@ -4,13 +4,12 @@ */ import { expire_time } from "@cocalc/util/misc"; -import { PostgreSQL } from "./types"; import { get_server_settings } from "./server-settings"; // this converts what's in the pii_expired setting to a new Date in the future export function pii_retention_to_future( pii_retention: number | false, - data?: T & { expire?: Date } + data?: T & { expire?: Date }, ): Date | undefined { if (!pii_retention) return; const future: Date = expire_time(pii_retention); @@ -25,9 +24,8 @@ export function pii_retention_to_future( // if data is set, it's expire field will be set. in any case, it returns the "Date" // in the future. export async function pii_expire( - db: PostgreSQL, - data?: T & { expire?: Date } + data?: T & { expire?: Date }, ): Promise { - const settings = await get_server_settings(db); + const settings = await get_server_settings(); return pii_retention_to_future(settings.pii_retention, data); } diff --git a/src/packages/database/postgres/server-settings.ts b/src/packages/database/postgres/server-settings.ts index d1da1a89fe..1cd45ccd32 100644 --- a/src/packages/database/postgres/server-settings.ts +++ b/src/packages/database/postgres/server-settings.ts @@ -1,10 +1,8 @@ -import { PostgreSQL } from "./types"; import { AllSiteSettings } from "@cocalc/util/db-schema/types"; import { callback2 } from "@cocalc/util/async-utils"; +import { db } from "@cocalc/database"; // just to make this async friendly, that's all -export async function get_server_settings( - db: PostgreSQL -): Promise { - return await callback2(db.get_server_settings_cached); +export async function get_server_settings(): Promise { + return await callback2(db().get_server_settings_cached); } diff --git a/src/packages/hub/analytics.ts b/src/packages/hub/analytics.ts index eaa426944c..936710c669 100644 --- a/src/packages/hub/analytics.ts +++ b/src/packages/hub/analytics.ts @@ -189,7 +189,7 @@ export async function initAnalytics( const dbg = create_log("analytics_js/cors"); // we only get the DNS once at startup – i.e. hub restart required upon changing DNS! - const settings = await get_server_settings(database); + const settings = await get_server_settings(); const DNS = settings.dns; const dns_parsed = parseDomain(DNS); const pii_retention = settings.pii_retention; diff --git a/src/packages/hub/servers/robots.ts b/src/packages/hub/servers/robots.ts index 56a6def796..0022fd6c3f 100644 --- a/src/packages/hub/servers/robots.ts +++ b/src/packages/hub/servers/robots.ts @@ -1,9 +1,8 @@ import { get_server_settings } from "@cocalc/database/postgres/server-settings"; -import { database } from "./database"; export default function getHandler() { return async (_req, res) => { - const settings = await get_server_settings(database); // don't worry -- this is cached. + const settings = await get_server_settings(); // don't worry -- this is cached. res.header("Content-Type", "text/plain"); res.header("Cache-Control", "public, max-age=3600, must-revalidate"); if (!settings.landing_pages) { diff --git a/src/packages/server/auth/auth-token.ts b/src/packages/server/auth/auth-token.ts index 917c5a3021..ca00846b49 100644 --- a/src/packages/server/auth/auth-token.ts +++ b/src/packages/server/auth/auth-token.ts @@ -2,6 +2,7 @@ import isAdmin from "@cocalc/server/accounts/is-admin"; import { generate } from "random-key"; import getPool from "@cocalc/database/pool"; import isPasswordCorrect from "./is-password-correct"; +import centralLog from "@cocalc/database/postgres/central-log"; // map {account_id:{user_account_id:timestamp}} const ban: { [account_id: string]: { [user_account_id: string]: number } } = {}; @@ -25,7 +26,8 @@ export async function generateUserAuthToken({ `banned -- please wait at least #{BAN_TIME_MS/1000}s before trying again`, ); } - if (!(await isAdmin(account_id))) { + const is_admin = await isAdmin(account_id); + if (!is_admin) { // not admin, so check password if (!(await isPasswordCorrect({ account_id: user_account_id, password }))) { if (ban[account_id] == null) { @@ -43,6 +45,10 @@ export async function generateUserAuthToken({ "INSERT INTO auth_tokens (auth_token, expire, account_id) VALUES($1, NOW()+INTERVAL '12 hours', $2)", [authToken, account_id], ); + await centralLog({ + event: "auth-token", + value: { account_id, user_account_id, is_admin }, + }); return authToken; } diff --git a/src/packages/server/mentions/handle.ts b/src/packages/server/mentions/handle.ts index ccbffb4c52..a5bed64713 100644 --- a/src/packages/server/mentions/handle.ts +++ b/src/packages/server/mentions/handle.ts @@ -5,7 +5,6 @@ Handle all mentions that haven't yet been handled. import { delay } from "awaiting"; import { getLogger } from "@cocalc/backend/logger"; -import { db } from "@cocalc/database"; import getPool from "@cocalc/database/pool"; import { pii_expire } from "@cocalc/database/postgres/pii"; import { expire_time } from "@cocalc/util/misc"; @@ -134,5 +133,5 @@ async function setError( // expire either after the PII setting or 1 year. async function getExpire(): Promise { - return (await pii_expire(db())) ?? expire_time(365 * 24 * 60 * 60); + return (await pii_expire()) ?? expire_time(365 * 24 * 60 * 60); } From acb2a1ad0a0f8ea9720c1ea5faedca0abc3937be Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 13:52:40 +0000 Subject: [PATCH 111/281] nats: switch admin user impersonate to new api --- src/packages/frontend/client/admin.ts | 21 +++++++--------- src/packages/frontend/client/client.ts | 4 +--- src/packages/hub/client.coffee | 24 ------------------- src/packages/util/message.js | 33 -------------------------- 4 files changed, 10 insertions(+), 72 deletions(-) diff --git a/src/packages/frontend/client/admin.ts b/src/packages/frontend/client/admin.ts index 6deb16ab77..00a30d7315 100644 --- a/src/packages/frontend/client/admin.ts +++ b/src/packages/frontend/client/admin.ts @@ -4,19 +4,19 @@ */ import * as message from "@cocalc/util/message"; -import { AsyncCall } from "./client"; +import type { WebappClient } from "./client"; import api from "./api"; export class AdminClient { - private async_call: AsyncCall; + private client: WebappClient; - constructor(async_call: AsyncCall) { - this.async_call = async_call; + constructor(client: WebappClient) { + this.client = client; } public async admin_reset_password(email_address: string): Promise { return ( - await this.async_call({ + await this.client.async_call({ message: message.admin_reset_password({ email_address, }), @@ -36,12 +36,9 @@ export class AdminClient { } } - public async get_user_auth_token(account_id: string): Promise { - return ( - await this.async_call({ - message: message.user_auth({ account_id, password: "" }), - allow_post: false, - }) - ).auth_token; + public async get_user_auth_token(user_account_id: string): Promise { + return await this.client.nats_client.hub.system.generateUserAuthToken({ + user_account_id, + }); } } diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 75175ef500..d039d2cdc3 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -218,9 +218,7 @@ class Client extends EventEmitter implements WebappClient { this.sync_string = this.sync_client.sync_string; this.sync_db = this.sync_client.sync_db; - this.admin_client = bind_methods( - new AdminClient(this.async_call.bind(this)), - ); + this.admin_client = bind_methods(new AdminClient(this)); this.openai_client = bind_methods(new LLMClient(this)); //this.purchases_client = bind_methods(new PurchasesClient(this)); this.purchases_client = bind_methods(new PurchasesClient()); diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee index 0ef3a12db3..f4c8def519 100644 --- a/src/packages/hub/client.coffee +++ b/src/packages/hub/client.coffee @@ -17,7 +17,6 @@ message = require('@cocalc/util/message') access = require('./access') clients = require('./clients').getClients() auth = require('./auth') -auth_token = require('./auth-token') local_hub_connection = require('./local_hub_connection') hub_projects = require('./projects') {StripeClient} = require('@cocalc/server/stripe/client') @@ -1592,29 +1591,6 @@ class exports.Client extends EventEmitter # END stripe-related functionality - mesg_user_auth: (mesg) => - auth_token.get_user_auth_token - database : @database - account_id : @account_id # strictly not necessary yet... but good if user has to be signed in, - # since more secure and we can rate limit attempts from a given user. - user_account_id : mesg.account_id - password : mesg.password - cb : (err, auth_token) => - if err - @error_to_client(id:mesg.id, error:err) - else - @push_to_client(message.user_auth_token(id:mesg.id, auth_token:auth_token)) - - mesg_revoke_auth_token: (mesg) => - auth_token.revoke_user_auth_token - database : @database - auth_token : mesg.auth_token - cb : (err) => - if err - @error_to_client(id:mesg.id, error:err) - else - @push_to_client(message.success(id:mesg.id)) - _check_project_access: (project_id, cb) => if not @account_id? cb('you must be signed in to access project') diff --git a/src/packages/util/message.js b/src/packages/util/message.js index abcf65dd33..8a2b9cf5c6 100644 --- a/src/packages/util/message.js +++ b/src/packages/util/message.js @@ -2448,39 +2448,6 @@ and provide that to a user. When they visit that URL, they will be temporarily }), ); -// hub --> client -message({ - event: "user_auth_token", - id: undefined, - auth_token: required, -}); // 24 character string - -/* -* Not fully implemented yet -* client --> hub -API message2 - event : 'revoke_auth_token' - fields: - id: - init : undefined - desc : 'A unique UUID for the query' - auth_token: - init : required - desc : 'an authentication token obtained using user_auth (24 character string)' - desc : """ -Example: - -Revoke a temporary authentication token for an account. -``` - curl -u sk_abcdefQWERTY090900000000: \\ - -d auth_token=BQokikJOvBiI2HlWgH4olfQ2 \\ - https://cocalc.com/api/v1/revoke_auth_token - ==> {"event":"success","id":"9e8b68ac-08e8-432a-a853-398042fae8c9"} -``` -""" -*/ - - // Info about available upgrades for a given user API( message2({ From bb57170585fe01abaf8f00a143bc219e8fca8c22 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 14:15:16 +0000 Subject: [PATCH 112/281] nats hub -- rewrite user search --- src/packages/frontend/client/client.ts | 4 +- src/packages/frontend/client/users.ts | 58 +++++++++++-------- .../frontend/frame-editors/generic/client.ts | 12 +--- src/packages/hub/client.coffee | 37 ------------ src/packages/nats/hub-api/system.ts | 10 ++++ src/packages/server/accounts/search.ts | 16 +---- src/packages/server/nats/api/system.ts | 32 ++++++++++ src/packages/util/db-schema/accounts.ts | 15 +++++ src/packages/util/message.js | 18 +----- 9 files changed, 97 insertions(+), 105 deletions(-) diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index d039d2cdc3..5eb1cf4970 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -225,9 +225,7 @@ class Client extends EventEmitter implements WebappClient { this.jupyter_client = bind_methods( new JupyterClient(this.async_call.bind(this)), ); - this.users_client = bind_methods( - new UsersClient(this.call.bind(this), this.async_call.bind(this)), - ); + this.users_client = bind_methods(new UsersClient(this)); this.tracking_client = bind_methods(new TrackingClient(this)); this.nats_client = bind_methods(new NatsClient(this)); this.file_client = bind_methods(new FileClient(this.async_call.bind(this))); diff --git a/src/packages/frontend/client/users.ts b/src/packages/frontend/client/users.ts index 5d9bb964de..1d5ce1ead6 100644 --- a/src/packages/frontend/client/users.ts +++ b/src/packages/frontend/client/users.ts @@ -3,48 +3,60 @@ * License: MS-RSL – see LICENSE.md for details */ -import { AsyncCall } from "./client"; import { User } from "../frame-editors/generic/client"; import { isChatBot, chatBotName } from "@cocalc/frontend/account/chatbot"; import api from "./api"; import TTL from "@isaacs/ttlcache"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import * as message from "@cocalc/util/message"; +import type { WebappClient } from "./client"; const nameCache = new TTL({ ttl: 60 * 1000 }); export class UsersClient { - private async_call: AsyncCall; + private client: WebappClient; - constructor(_call: Function, async_call: AsyncCall) { - this.async_call = async_call; + constructor(client) { + this.client = client; } + /* + There are two possible item types in the query list: email addresses + and strings that are not email addresses. An email query item will return + account id, first name, last name, and email address for the unique + account with that email address, if there is one. A string query item + will return account id, first name, and last name for all matching + accounts. + + We do not reveal email addresses of users queried by name to non admins. + + String query matches first and last names that start with the given string. + If a string query item consists of two strings separated by space, + the search will return accounts in which the first name begins with one + of the two strings and the last name begins with the other. + String and email queries may be mixed in the list for a single + user_search call. Searches are case-insensitive. + + Note: there is a hard limit of 50 returned items in the results, except for + admins that can search for more. + */ user_search = reuseInFlight( - async (opts: { + async ({ + query, + limit = 20, + admin, + only_email, + }: { query: string; limit?: number; - active?: string; // if given, would restrict to users active this recently admin?: boolean; // admins can do an admin version of the query, which also does substring searches on email address (not just name) only_email?: boolean; // search only via email address }): Promise => { - if (opts.limit == null) { - opts.limit = 20; - } - if (opts.active == null) { - opts.active = ""; - } - - const { results } = await this.async_call({ - message: message.user_search({ - query: opts.query, - limit: opts.limit, - admin: opts.admin, - active: opts.active, - only_email: opts.only_email, - }), + return await this.client.nats_client.hub.system.userSearch({ + query, + limit, + admin, + only_email, }); - return results; }, ); diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index c63046fb70..95a597dd5e 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -15,6 +15,8 @@ import { CompressedPatch } from "@cocalc/sync/editor/generic/types"; import { callback2 } from "@cocalc/util/async-utils"; import { Config as FormatterConfig } from "@cocalc/util/code-formatter"; import { FakeSyncstring } from "./syncstring-fake"; +import { type UserSearchResult as User } from "@cocalc/util/db-schema/accounts"; +export { type User }; import type { ExecOpts, ExecOutput } from "@cocalc/util/db-schema/projects"; export type { ExecOpts, ExecOutput }; @@ -252,16 +254,6 @@ export function get_editor_settings(): Map { return Map(); // not loaded } -export interface User { - account_id: string; - created?: number; // since commit 63e8e9954dc51632cf - email_address?: string; - first_name?: string; - last_active?: number; // since commit 63e8e9954dc51632cf - last_name?: string; - banned?: boolean; -} - export async function user_search(opts: { query: string; limit?: number; diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee index f4c8def519..9e90c1735f 100644 --- a/src/packages/hub/client.coffee +++ b/src/packages/hub/client.coffee @@ -31,7 +31,6 @@ passwordHash = require("@cocalc/backend/auth/password-hash").default; jupyter_execute = require('@cocalc/server/jupyter/execute').execute; jupyter_kernels = require('@cocalc/server/jupyter/kernels').default; create_project = require("@cocalc/server/projects/create").default; -user_search = require("@cocalc/server/accounts/search").default; collab = require('@cocalc/server/projects/collab'); delete_passport = require('@cocalc/server/auth/sso/delete-passport').delete_passport; setEmailAddress = require("@cocalc/server/accounts/set-email-address").default; @@ -938,42 +937,6 @@ class exports.Client extends EventEmitter resp.id = mesg.id @push_to_client(resp) - mesg_user_search: (mesg) => - if not @account_id? - @push_to_client(message.error(id:mesg.id, error:"You must be signed in to search for users.")) - return - - if not mesg.admin and (not mesg.limit? or mesg.limit > 50) - # hard cap at 50... (for non-admin) - mesg.limit = 50 - locals = {results: undefined} - async.series([ - (cb) => - if mesg.admin - @assert_user_is_in_group('admin', cb) - else - cb() - (cb) => - @touch() - opts = - query : mesg.query - limit : mesg.limit - admin : mesg.admin - active : mesg.active - only_email: mesg.only_email - try - locals.results = await user_search(opts) - cb(undefined) - catch err - cb(err) - ], (err) => - if err - @error_to_client(id:mesg.id, error:err) - else - @push_to_client(message.user_search_results(id:mesg.id, results:locals.results)) - ) - - # this is an async function allow_urls_in_emails: (project_id) => is_paying = await is_paying_customer(@database, @account_id) diff --git a/src/packages/nats/hub-api/system.ts b/src/packages/nats/hub-api/system.ts index e96d3c1c35..908438e4e4 100644 --- a/src/packages/nats/hub-api/system.ts +++ b/src/packages/nats/hub-api/system.ts @@ -4,6 +4,7 @@ import type { ApiKey, Action as ApiKeyAction, } from "@cocalc/util/db-schema/api-keys"; +import { type UserSearchResult } from "@cocalc/util/db-schema/accounts"; export const system = { getCustomize: noAuth, @@ -14,6 +15,7 @@ export const system = { manageApiKeys: authFirst, generateUserAuthToken: authFirst, revokeUserAuthToken: noAuth, + userSearch: authFirst, }; export interface System { @@ -50,4 +52,12 @@ export interface System { }) => Promise; revokeUserAuthToken: (authToken: string) => Promise; + + userSearch: (opts: { + account_id?: string; + query: string; + limit?: number; + admin?: boolean; + only_email?: boolean; + }) => Promise; } diff --git a/src/packages/server/accounts/search.ts b/src/packages/server/accounts/search.ts index 08aa6ea829..ebe108b6c1 100644 --- a/src/packages/server/accounts/search.ts +++ b/src/packages/server/accounts/search.ts @@ -19,25 +19,11 @@ import { getLogger } from "@cocalc/backend/logger"; import { USER_SEARCH_LIMIT, ADMIN_SEARCH_LIMIT, + type UserSearchResult as User, } from "@cocalc/util/db-schema/accounts"; const logger = getLogger("accounts/search"); -export interface User { - account_id: string; - first_name?: string; - last_name?: string; - name?: string; // "vanity" username - last_active?: number; // ms since epoch -- when account was last active - created?: number; // ms since epoch -- when account created - banned?: boolean; // true if this user has been banned (only set for admin searches, obviously) - email_address_verified?: boolean; // true if their email has been verified (a sign they are more trustworthy). - // For security reasons, the email_address *only* occurs in search queries that - // are by email_address (or for admins); we must not reveal email addresses - // of users queried by substring searches, obviously. - email_address?: string; -} - interface DBUser { account_id: string; first_name?: string; diff --git a/src/packages/server/nats/api/system.ts b/src/packages/server/nats/api/system.ts index 9719e05fa8..89639e7d22 100644 --- a/src/packages/server/nats/api/system.ts +++ b/src/packages/server/nats/api/system.ts @@ -6,6 +6,9 @@ import { record_user_tracking } from "@cocalc/database/postgres/user-tracking"; import { db } from "@cocalc/database"; import manageApiKeys from "@cocalc/server/api/manage"; export { manageApiKeys }; +import { type UserSearchResult } from "@cocalc/util/db-schema/accounts"; +import isAdmin from "@cocalc/server/accounts/is-admin"; +import search from "@cocalc/server/accounts/search"; export function ping() { return { now: Date.now() }; @@ -29,3 +32,32 @@ export { generateUserAuthToken, revokeUserAuthToken, } from "@cocalc/server/auth/auth-token"; + +export async function userSearch({ + account_id, + query, + limit, + admin, + only_email, +}: { + account_id?: string; + query: string; + limit?: number; + admin?: boolean; + only_email?: boolean; +}): Promise { + if (!account_id) { + throw Error("You must be signed in to search for users."); + } + if (admin) { + if (!(await isAdmin(account_id))) { + throw Error("Must be an admin to do admin search."); + } + } else { + if (limit != null && limit > 50) { + // hard cap at 50... (for non-admin) + limit = 50; + } + } + return await search({ query, limit, admin, only_email }); +} diff --git a/src/packages/util/db-schema/accounts.ts b/src/packages/util/db-schema/accounts.ts index 8914b0b4bb..58b7ddf815 100644 --- a/src/packages/util/db-schema/accounts.ts +++ b/src/packages/util/db-schema/accounts.ts @@ -902,3 +902,18 @@ for (const x of TAGS) { export const CONTACT_TAG = "contact"; export const CONTACT_THESE_TAGS = [professional]; + +export interface UserSearchResult { + account_id: string; + first_name?: string; + last_name?: string; + name?: string; // "vanity" username + last_active?: number; // ms since epoch -- when account was last active + created?: number; // ms since epoch -- when account created + banned?: boolean; // true if this user has been banned (only set for admin searches, obviously) + email_address_verified?: boolean; // true if their email has been verified (a sign they are more trustworthy). + // For security reasons, the email_address *only* occurs in search queries that + // are by email_address (or for admins); we must not reveal email addresses + // of users queried by substring searches, obviously. + email_address?: string; +} diff --git a/src/packages/util/message.js b/src/packages/util/message.js index 8a2b9cf5c6..305cffe42d 100644 --- a/src/packages/util/message.js +++ b/src/packages/util/message.js @@ -1051,23 +1051,7 @@ API( }, }, desc: `\ -There are two possible item types in the query list: email addresses -and strings that are not email addresses. An email query item will return -account id, first name, last name, and email address for the unique -account with that email address, if there is one. A string query item -will return account id, first name, and last name for all matching -accounts. - -We do not reveal email addresses of users queried by name to non admins. - -String query matches first and last names that start with the given string. -If a string query item consists of two strings separated by space, -the search will return accounts in which the first name begins with one -of the two strings and the last name begins with the other. -String and email queries may be mixed in the list for a single -user_search call. Searches are case-insensitive. - -Note: there is a hard limit of 50 returned items in the results. + Examples: From 2df7477ae544827253b8079186fa8a7f50f86677 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 14:25:49 +0000 Subject: [PATCH 113/281] nats: rewrite getNames to use new api --- src/packages/frontend/client/users.ts | 9 ++++++--- src/packages/nats/hub-api/system.ts | 13 ++++++++++++- src/packages/nats/hub-api/util.ts | 8 ++++++++ src/packages/server/nats/api/system.ts | 1 + 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/packages/frontend/client/users.ts b/src/packages/frontend/client/users.ts index 1d5ce1ead6..4598ef9e7e 100644 --- a/src/packages/frontend/client/users.ts +++ b/src/packages/frontend/client/users.ts @@ -5,7 +5,6 @@ import { User } from "../frame-editors/generic/client"; import { isChatBot, chatBotName } from "@cocalc/frontend/account/chatbot"; -import api from "./api"; import TTL from "@isaacs/ttlcache"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import type { WebappClient } from "./client"; @@ -84,7 +83,11 @@ export class UsersClient { getNames = reuseInFlight(async (account_ids: string[]) => { const x: { [account_id: string]: - | { first_name: string; last_name: string } + | { + first_name: string; + last_name: string; + profile?: { color?: string; image?: string }; + } | undefined; } = {}; const v: string[] = []; @@ -96,7 +99,7 @@ export class UsersClient { } } if (v.length > 0) { - const { names } = await api("/accounts/get-names", { account_ids: v }); + const names = await this.client.nats_client.hub.system.getNames(v); for (const account_id of v) { // iterate over v to record accounts that don't exist too x[account_id] = names[account_id]; diff --git a/src/packages/nats/hub-api/system.ts b/src/packages/nats/hub-api/system.ts index 908438e4e4..e58a1fd05f 100644 --- a/src/packages/nats/hub-api/system.ts +++ b/src/packages/nats/hub-api/system.ts @@ -1,4 +1,4 @@ -import { noAuth, authFirst } from "./util"; +import { noAuth, authFirst, requireAccount } from "./util"; import type { Customize } from "@cocalc/util/db-schema/server-settings"; import type { ApiKey, @@ -16,6 +16,7 @@ export const system = { generateUserAuthToken: authFirst, revokeUserAuthToken: noAuth, userSearch: authFirst, + getNames: requireAccount, }; export interface System { @@ -60,4 +61,14 @@ export interface System { admin?: boolean; only_email?: boolean; }) => Promise; + + getNames: (account_ids: string[]) => Promise<{ + [account_id: string]: + | { + first_name: string; + last_name: string; + profile?: { color?: string; image?: string }; + } + | undefined; + }>; } diff --git a/src/packages/nats/hub-api/util.ts b/src/packages/nats/hub-api/util.ts index 8c3724536f..48a2a8e88a 100644 --- a/src/packages/nats/hub-api/util.ts +++ b/src/packages/nats/hub-api/util.ts @@ -11,3 +11,11 @@ export const authFirst = ({ args, account_id, project_id }) => { }; export const noAuth = ({ args }) => args; + +// make no changes, except throw error if account_id not set (i.e., user not signed in) +export const requireAccount = ({ args, account_id }) => { + if (!account_id) { + throw Error("user must be signed in"); + } + return args; +}; diff --git a/src/packages/server/nats/api/system.ts b/src/packages/server/nats/api/system.ts index 89639e7d22..40b062677b 100644 --- a/src/packages/server/nats/api/system.ts +++ b/src/packages/server/nats/api/system.ts @@ -9,6 +9,7 @@ export { manageApiKeys }; import { type UserSearchResult } from "@cocalc/util/db-schema/accounts"; import isAdmin from "@cocalc/server/accounts/is-admin"; import search from "@cocalc/server/accounts/search"; +export { getNames } from "@cocalc/server/accounts/get-name"; export function ping() { return { now: Date.now() }; From fa63cdea8cefe0102ab46fd1aa6e9ffd28bed8de Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 14:54:13 +0000 Subject: [PATCH 114/281] nats: user new api for getBalance --- src/packages/frontend/client/client.ts | 3 +-- src/packages/frontend/client/purchases.ts | 7 +++++-- src/packages/frontend/purchases/balance-button.tsx | 8 +------- src/packages/frontend/purchases/balance-modal.tsx | 4 ++-- src/packages/nats/hub-api/purchases.ts | 12 ++++++------ src/packages/server/accounts/search.ts | 1 + src/packages/server/nats/api/purchases.ts | 8 ++++++-- 7 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 5eb1cf4970..4428815a4b 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -220,8 +220,7 @@ class Client extends EventEmitter implements WebappClient { this.admin_client = bind_methods(new AdminClient(this)); this.openai_client = bind_methods(new LLMClient(this)); - //this.purchases_client = bind_methods(new PurchasesClient(this)); - this.purchases_client = bind_methods(new PurchasesClient()); + this.purchases_client = bind_methods(new PurchasesClient(this)); this.jupyter_client = bind_methods( new JupyterClient(this.async_call.bind(this)), ); diff --git a/src/packages/frontend/client/purchases.ts b/src/packages/frontend/client/purchases.ts index 4878734a43..f247775d9f 100644 --- a/src/packages/frontend/client/purchases.ts +++ b/src/packages/frontend/client/purchases.ts @@ -14,12 +14,15 @@ import type { ProjectQuota } from "@cocalc/util/db-schema/purchase-quotas"; import * as purchasesApi from "@cocalc/frontend/purchases/api"; import type { Changes as EditLicenseChanges } from "@cocalc/util/purchases/cost-to-edit-license"; import { round2up } from "@cocalc/util/misc"; +import type { WebappClient } from "./client"; export class PurchasesClient { api: typeof purchasesApi; + client: WebappClient; - constructor() { + constructor(client: WebappClient) { this.api = purchasesApi; + this.client = client; } async getQuotas(): Promise<{ minBalance: number; @@ -29,7 +32,7 @@ export class PurchasesClient { } async getBalance(): Promise { - return await purchasesApi.getBalance(); + return await this.client.nats_client.hub.purchases.getBalance(); } async getSpendRate(): Promise { diff --git a/src/packages/frontend/purchases/balance-button.tsx b/src/packages/frontend/purchases/balance-button.tsx index 4c726e84f5..0ffcd851ea 100644 --- a/src/packages/frontend/purchases/balance-button.tsx +++ b/src/packages/frontend/purchases/balance-button.tsx @@ -1,7 +1,6 @@ import { Badge, Button, Spin } from "antd"; import { useEffect, useState } from "react"; import { useIntl } from "react-intl"; - import { CSS, useTypedRedux } from "@cocalc/frontend/app-framework"; import { NavTab } from "@cocalc/frontend/app/nav-tab"; import { NAV_CLASS } from "@cocalc/frontend/app/top-nav-consts"; @@ -9,7 +8,6 @@ import { labels } from "@cocalc/frontend/i18n"; import BalanceModal from "@cocalc/frontend/purchases/balance-modal"; import { webapp_client } from "@cocalc/frontend/webapp-client"; import { currency, round2down } from "@cocalc/util/misc"; -import { getBalance as getBalanceUsingApi } from "./api"; export default function BalanceButton({ style, @@ -36,10 +34,6 @@ export default function BalanceButton({ } }, [dbBalance]); - const getBalance = async () => { - setBalance(await getBalanceUsingApi()); - }; - const handleRefresh = async () => { if (!webapp_client.account_id) { // not signed in. @@ -48,7 +42,7 @@ export default function BalanceButton({ try { onRefresh?.(); setLoading(true); - await getBalance(); + await webapp_client.purchases_client.getBalance(); } catch (err) { console.warn("Issue updating balance", err); } finally { diff --git a/src/packages/frontend/purchases/balance-modal.tsx b/src/packages/frontend/purchases/balance-modal.tsx index 8d801df95e..84d0b15a42 100644 --- a/src/packages/frontend/purchases/balance-modal.tsx +++ b/src/packages/frontend/purchases/balance-modal.tsx @@ -1,12 +1,12 @@ import { Button, Flex, Modal, Space, Spin } from "antd"; import Balance from "./balance"; import { useEffect, useRef, useState } from "react"; -import { getBalance as getBalanceUsingApi } from "./api"; import ShowError from "@cocalc/frontend/components/error"; import { redux } from "@cocalc/frontend/app-framework"; import Payments from "@cocalc/frontend/purchases/payments"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import { join } from "path"; +import { webapp_client } from "@cocalc/frontend/webapp-client"; export default function BalanceModal({ onRefresh, @@ -25,7 +25,7 @@ export default function BalanceModal({ setError(""); setLoading(true); // this triggers an update indirectly - await getBalanceUsingApi(); + await webapp_client.purchases_client.getBalance(); await refreshPaymentsRef.current?.(); } catch (err) { setError(`${err}`); diff --git a/src/packages/nats/hub-api/purchases.ts b/src/packages/nats/hub-api/purchases.ts index 697160717f..eb98d0c882 100644 --- a/src/packages/nats/hub-api/purchases.ts +++ b/src/packages/nats/hub-api/purchases.ts @@ -1,11 +1,11 @@ +import { authFirst } from "./util"; + export interface Purchases { - getBalance: ({ account_id }) => Promise; - getMinBalance: (account_id) => Promise; + getBalance: (opts?: { account_id?: string }) => Promise; + getMinBalance: (opts?: { account_id?: string }) => Promise; } export const purchases = { - getBalance: ({ account_id }) => { - return [{ account_id }]; - }, - getMinBalance: ({ account_id }) => [account_id], + getBalance: authFirst, + getMinBalance: authFirst, }; diff --git a/src/packages/server/accounts/search.ts b/src/packages/server/accounts/search.ts index ebe108b6c1..e4dc677af6 100644 --- a/src/packages/server/accounts/search.ts +++ b/src/packages/server/accounts/search.ts @@ -21,6 +21,7 @@ import { ADMIN_SEARCH_LIMIT, type UserSearchResult as User, } from "@cocalc/util/db-schema/accounts"; +export { type User }; const logger = getLogger("accounts/search"); diff --git a/src/packages/server/nats/api/purchases.ts b/src/packages/server/nats/api/purchases.ts index 38831edb5e..d81538f88f 100644 --- a/src/packages/server/nats/api/purchases.ts +++ b/src/packages/server/nats/api/purchases.ts @@ -1,4 +1,8 @@ import getBalance from "@cocalc/server/purchases/get-balance"; -import getMinBalance from "@cocalc/server/purchases/get-min-balance"; +import getMinBalance0 from "@cocalc/server/purchases/get-min-balance"; -export { getBalance, getMinBalance }; +export { getBalance }; + +export async function getMinBalance({ account_id }) { + return await getMinBalance0(account_id); +} From 867f35051779b7bd6e1561d2e63130fa938661a1 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 15:09:52 +0000 Subject: [PATCH 115/281] oops -- metrics-recorder is of course used for more than just browser metrics --- src/packages/hub/metrics-recorder.coffee | 181 +++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 src/packages/hub/metrics-recorder.coffee diff --git a/src/packages/hub/metrics-recorder.coffee b/src/packages/hub/metrics-recorder.coffee new file mode 100644 index 0000000000..aaacdbd251 --- /dev/null +++ b/src/packages/hub/metrics-recorder.coffee @@ -0,0 +1,181 @@ +######################################################################### +# This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. +# License: MS-RSL – see LICENSE.md for details +######################################################################### + +# This is a small helper class to record real-time metrics about the hub. +# It is designed for the hub, such that a local process can easily check its health. +# After an initial version, this has been repurposed to use prometheus. +# It wraps its client elements and adds some instrumentation to some hub components. + +fs = require('fs') +path = require('path') +underscore = require('underscore') +{execSync} = require('child_process') +{defaults} = misc = require('@cocalc/util/misc') + +# Prometheus client setup -- https://github.com/siimon/prom-client +prom_client = require('prom-client') + +# some constants +FREQ_s = 5 # update stats every FREQ seconds +DELAY_s = 10 # with an initial delay of DELAY seconds + +# collect some recommended default metrics +prom_client.collectDefaultMetrics(timeout: FREQ_s * 1000) + +# CLK_TCK (usually 100, but maybe not ...) +try + CLK_TCK = parseInt(execSync('getconf CLK_TCK', {encoding: 'utf8'})) +catch err + CLK_TCK = null + +### +# there is more than just continuous values +# cont: continuous (like number of changefeeds), will be smoothed +# disc: discrete, like blocked, will be recorded with timestamp +# in a queue of length DISC_LEN +exports.TYPE = TYPE = + COUNT: 'counter' # strictly non-decrasing integer + GAUGE: 'gauge' # only the most recent value is recorded + LAST : 'latest' # only the most recent value is recorded + DISC : 'discrete' # timeseries of length DISC_LEN + CONT : 'continuous' # continuous with exponential decay + MAX : 'contmax' # like CONT, reduces buffer to max value + SUM : 'contsum' # like CONT, reduces buffer to sum of values divided by FREQ_s +### + +PREFIX = 'cocalc_hub_' + +exports.new_counter = new_counter = (name, help, labels) -> + # a prometheus counter -- https://github.com/siimon/prom-client#counter + # use it like counter.labels(labelA, labelB).inc([positive number or default is 1]) + if not name.endsWith('_total') + throw "Counter metric names have to end in [_unit]_total but I got '#{name}' -- https://prometheus.io/docs/practices/naming/" + return new prom_client.Counter(name: PREFIX + name, help: help, labelNames: labels ? []) + +exports.new_gauge = new_gauge = (name, help, labels) -> + # a prometheus gauge -- https://github.com/siimon/prom-client#gauge + # basically, use it like gauge.labels(labelA, labelB).set(value) + return new prom_client.Gauge(name: PREFIX + name, help: help, labelNames: labels ? []) + +exports.new_quantile = new_quantile = (name, help, config={}) -> + # invoked as quantile.observe(value) + config = defaults config, + # a few more than the default, in particular including the actual min and max + percentiles: [0.0, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 0.999, 1.0] + labels : [] + return new prom_client.Summary(name: PREFIX + name, help: help, labelNames:config.labels, percentiles: config.percentiles) + +exports.new_histogram = new_histogram = (name, help, config={}) -> + # invoked as histogram.observe(value) + config = defaults config, + buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] + labels: [] + return new prom_client.Histogram(name: PREFIX + name, help: help, labelNames: config.labels, buckets:config.buckets) + + +# This is modified by the Client class (in client.coffee) when metrics +# get pushed from browsers. It's a map from client_id to +# an array of metrics objects, which are already labeled with extra +# information about the client_id and account_id. +exports.client_metrics = {} + +class MetricsRecorder + constructor: (@dbg, cb) -> + ### + * @dbg: reporting via winston, instance with configuration passed in from hub.coffee + ### + # stores the current state of the statistics + @_stats = {} + @_types = {} # key → TYPE.T mapping + + # the full statistic + @_data = {} + @_collectors = [] + + # initialization finished + @setup_monitoring() + cb?(undefined, @) + + client_metrics: => + ### + exports.client_metrics is a mapping of client id to the json exported metric. + The AggregatorRegistry is supposed to work with a list of metrics, and by default, + it sums them up. `aggregate` is a static method and hence it should be ok to use it directly. + ### + metrics = (m for _, m of exports.client_metrics) + + registry = prom_client.AggregatorRegistry.aggregate(metrics) + return await registry.metrics() + + metrics: => + ### + get a serialized representation of the metrics status + (was a dict that should be JSON, now it is for prometheus) + it's only called by the HTTP stuff in servers for the /metrics endpoint + ### + hub = await prom_client.register.metrics() + clients = await @client_metrics() + return hub + clients + + register_collector: (collector) => + # The added collector functions will be evaluated periodically to gather metrics + @_collectors.push(collector) + + setup_monitoring: => + # setup monitoring of some components + # called by the hub *after* setting up the DB, etc. + num_clients_gauge = new_gauge('clients_count', 'Number of connected clients') + {number_of_clients} = require('./hub_register') + @register_collector -> + try + num_clients_gauge.set(number_of_clients()) + catch + num_clients_gauge.set(0) + + + # our own CPU metrics monitor, separating user and sys! + # it's actually a counter, since it is non-decreasing, but we'll use .set(...) + @_cpu_seconds_total = new_gauge('process_cpu_categorized_seconds_total', 'Total number of CPU seconds used', ['type']) + + @_collect_duration = new_histogram('metrics_collect_duration_s', 'How long it took to gather the metrics', buckets:[0.0001, 0.001, 0.01, 1]) + @_collect_duration_last = new_gauge('metrics_collect_duration_s_last', 'How long it took the last time to gather the metrics') + + # init periodically calling @_collect + setTimeout((=> setInterval(@_collect, FREQ_s * 1000)), DELAY_s * 1000) + + _collect: => + endG = @_collect_duration_last.startTimer() + endH = @_collect_duration.startTimer() + + # called by @_update to evaluate the collector functions + #@dbg('_collect called') + for c in @_collectors + c() + # linux specific: collecting this process and all its children sys+user times + # http://man7.org/linux/man-pages/man5/proc.5.html + fs.readFile path.join('/proc', ''+process.pid, 'stat'), 'utf8', (err, infos) => + if err or not CLK_TCK? + @dbg("_collect err: #{err}") + return + # there might be spaces in the process name, hence split after the closing bracket! + infos = infos[infos.lastIndexOf(')') + 2...].split(' ') + @_cpu_seconds_total.labels('user') .set(parseFloat(infos[11]) / CLK_TCK) + @_cpu_seconds_total.labels('system') .set(parseFloat(infos[12]) / CLK_TCK) + # time spent waiting on child processes + @_cpu_seconds_total.labels('chld_user') .set(parseFloat(infos[13]) / CLK_TCK) + @_cpu_seconds_total.labels('chld_system').set(parseFloat(infos[14]) / CLK_TCK) + + # END: the timings for this run. + endG() + endH() + +metricsRecorder = null +exports.init = (winston, cb) -> + dbg = (msg) -> + winston.info("MetricsRecorder: #{msg}") + metricsRecorder = new MetricsRecorder(dbg, cb) + +exports.get = -> + return metricsRecorder From 76a6610368b4b56f2c57baafbf25e886fdbbb096 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 16:47:49 +0000 Subject: [PATCH 116/281] nats: new api for create project --- .../frontend/account/account-page.tsx | 1 - src/packages/frontend/client/nats.ts | 2 +- src/packages/frontend/client/project.ts | 14 +- .../frontend/project/explorer/explorer.tsx | 1 - .../explorer/fetch-directory-errors.tsx | 4 +- .../frontend/project/websocket/api.ts | 2 +- src/packages/hub/client.coffee | 57 ------- src/packages/nats/hub-api/index.ts | 3 + src/packages/nats/hub-api/projects.ts | 13 ++ src/packages/nats/hub-api/system.ts | 3 - src/packages/nats/hub-api/util.ts | 12 ++ src/packages/server/nats/api/index.ts | 10 +- src/packages/server/nats/api/projects.ts | 5 + src/packages/server/nats/api/system.ts | 2 - src/packages/server/projects/create.ts | 50 +++++-- src/packages/util/db-schema/projects.ts | 16 ++ src/packages/util/message.d.ts | 4 - src/packages/util/message.js | 140 ------------------ 18 files changed, 104 insertions(+), 235 deletions(-) create mode 100644 src/packages/nats/hub-api/projects.ts create mode 100644 src/packages/server/nats/api/projects.ts diff --git a/src/packages/frontend/account/account-page.tsx b/src/packages/frontend/account/account-page.tsx index 4a0427ee54..6d649d944c 100644 --- a/src/packages/frontend/account/account-page.tsx +++ b/src/packages/frontend/account/account-page.tsx @@ -15,7 +15,6 @@ and configuration. import { Flex, Menu, Space } from "antd"; import { useEffect } from "react"; import { useIntl } from "react-intl"; - import { SignOut } from "@cocalc/frontend/account/sign-out"; import { React, diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 2ca5043343..5ff5625275 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -153,7 +153,7 @@ export class NatsClient { } catch (err) { if (err.code == "PERMISSIONS_VIOLATION") { // request update of our credentials to include this project, then try again - await this.hub.system.addProjectPermission({ project_id }); + await this.hub.projects.addProjectPermission({ project_id }); resp = await nc.request(subject, mesg, { timeout }); } else { throw err; diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index f9b83c56e4..01bbf0c517 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -502,19 +502,17 @@ export class ProjectClient { description: string; image?: string; start?: boolean; - license?: string; // "license_id1,license_id2,..." -- if given, create project with these licenses applied - noPool?: boolean; // never use pool + // "license_id1,license_id2,..." -- if given, create project with these licenses applied + license?: string; + // never use pool + noPool?: boolean; }): Promise { - const { project_id } = await this.client.async_call({ - allow_post: false, // since gets called for anonymous and cookie not yet set. - message: message.create_project(opts), - }); - + const project_id = + await this.client.nats_client.hub.projects.createProject(opts); this.client.tracking_client.user_tracking("create_project", { project_id, title: opts.title, }); - return project_id; } diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 77603c6bbc..f04e7498b1 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -451,7 +451,6 @@ const Explorer0 = rclass( quotas={this.props.get_total_project_quotas( this.props.project_id, )} - is_commercial={require("@cocalc/frontend/customize").commercial} is_logged_in={!!this.props.is_logged_in} />
diff --git a/src/packages/frontend/project/explorer/fetch-directory-errors.tsx b/src/packages/frontend/project/explorer/fetch-directory-errors.tsx index 5305797ea4..3e50ec8834 100644 --- a/src/packages/frontend/project/explorer/fetch-directory-errors.tsx +++ b/src/packages/frontend/project/explorer/fetch-directory-errors.tsx @@ -5,12 +5,12 @@ import ShowError from "@cocalc/frontend/components/error"; import { AccessErrors } from "./access-errors"; +import { useTypedRedux } from "@cocalc/frontend/app-framework"; interface Props { error: any; path: string; quotas: any; - is_commercial: boolean; is_logged_in: boolean; } @@ -18,9 +18,9 @@ export function FetchDirectoryErrors({ error, path, quotas, - is_commercial, is_logged_in, }: Props): JSX.Element { + const is_commercial = useTypedRedux("customize", "is_commercial"); switch (error) { case "not_public": return ; diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index ef320dca10..668330068d 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -94,7 +94,7 @@ export class API { } catch (err) { if (err.code == "PERMISSIONS_VIOLATION") { // request update of our credentials to include this project, then try again - await webapp_client.nats_client.hub.system.addProjectPermission({ + await webapp_client.nats_client.hub.projects.addProjectPermission({ project_id: this.project_id, }); return await this._call(mesg, timeout); diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee index 9e90c1735f..421b0a896b 100644 --- a/src/packages/hub/client.coffee +++ b/src/packages/hub/client.coffee @@ -755,63 +755,6 @@ class exports.Client extends EventEmitter cb(undefined, project) ) - mesg_create_project: (mesg) => - if not @account_id? - @error_to_client(id: mesg.id, error: "You must be signed in to create a new project.") - return - @touch() - - dbg = @dbg('mesg_create_project') - - project_id = undefined - project = undefined - - async.series([ - (cb) => - dbg("create project entry in database") - try - opts = - account_id : @account_id - title : mesg.title - description : mesg.description - image : mesg.image - license : mesg.license - noPool : mesg.noPool - project_id = await create_project(opts) - cb(undefined) - catch err - cb(err) - (cb) => - cb() # we don't need to wait for project to start running before responding to user that project was created. - dbg("open project...") - # We do the open/start below so that when user tries to open it in a moment it opens more quickly; - # also, in single dev mode, this ensures that project path is created, so can copy - # files to the project, etc. - # Also, if mesg.start is set, the project gets started below. - try - project = await @projectControl(project_id) - await project.state(force:true, update:true) - if mesg.start - await project.start() - await delay(5000) # just in case - await project.start() - else - dbg("not auto-starting the new project") - catch err - dbg("failed to start project running -- #{err}") - ], (err) => - if err - dbg("error; project #{project_id} -- #{err}") - @error_to_client(id: mesg.id, error: "Failed to create new project '#{mesg.title}' -- #{misc.to_json(err)}") - else - dbg("SUCCESS: project #{project_id}") - @push_to_client(message.project_created(id:mesg.id, project_id:project_id)) - # As an optimization, we start the process of opening the project, since the user is likely - # to open the project soon anyways. - dbg("start process of opening project") - @get_project {project_id:project_id}, 'write', (err, project) => - ) - mesg_write_text_file_to_project: (mesg) => @get_project mesg, 'write', (err, project) => if err diff --git a/src/packages/nats/hub-api/index.ts b/src/packages/nats/hub-api/index.ts index 4861155bbf..5f3cf88f58 100644 --- a/src/packages/nats/hub-api/index.ts +++ b/src/packages/nats/hub-api/index.ts @@ -1,12 +1,14 @@ import { isValidUUID } from "@cocalc/util/misc"; import { type Purchases, purchases } from "./purchases"; import { type System, system } from "./system"; +import { type Projects, projects } from "./projects"; import { type DB, db } from "./db"; import { type LLM, llm } from "./llm"; import { handleErrorMessage } from "@cocalc/nats/util"; export interface HubApi { system: System; + projects: Projects; db: DB; purchases: Purchases; llm: LLM; @@ -14,6 +16,7 @@ export interface HubApi { const HubApiStructure = { system, + projects, db, purchases, llm, diff --git a/src/packages/nats/hub-api/projects.ts b/src/packages/nats/hub-api/projects.ts new file mode 100644 index 0000000000..13b892be6d --- /dev/null +++ b/src/packages/nats/hub-api/projects.ts @@ -0,0 +1,13 @@ +import { authFirstRequireAccount } from "./util"; +import { type CreateProjectOptions } from "@cocalc/util/db-schema/projects"; + +export const projects = { + addProjectPermission: authFirstRequireAccount, + createProject: authFirstRequireAccount, +}; + +export interface Projects { + // request to have NATS permissions to project subjects. + addProjectPermission: (opts: { project_id: string }) => Promise; + createProject: (opts: CreateProjectOptions) => Promise; +} diff --git a/src/packages/nats/hub-api/system.ts b/src/packages/nats/hub-api/system.ts index e58a1fd05f..8f590c16b4 100644 --- a/src/packages/nats/hub-api/system.ts +++ b/src/packages/nats/hub-api/system.ts @@ -9,7 +9,6 @@ import { type UserSearchResult } from "@cocalc/util/db-schema/accounts"; export const system = { getCustomize: noAuth, ping: noAuth, - addProjectPermission: authFirst, terminate: authFirst, userTracking: authFirst, manageApiKeys: authFirst, @@ -24,8 +23,6 @@ export interface System { getCustomize: (fields?: string[]) => Promise; // ping server and get back the current time ping: () => { now: number }; - // request to have NATS permissions to project subjects. - addProjectPermission: (opts: { project_id: string }) => Promise; // terminate a service: // - only admin can do this. // - useful for development diff --git a/src/packages/nats/hub-api/util.ts b/src/packages/nats/hub-api/util.ts index 48a2a8e88a..ff7e896514 100644 --- a/src/packages/nats/hub-api/util.ts +++ b/src/packages/nats/hub-api/util.ts @@ -19,3 +19,15 @@ export const requireAccount = ({ args, account_id }) => { } return args; }; + +export const authFirstRequireAccount = async ({ args, account_id }) => { + if (args[0] == null) { + args[0] = {} as any; + } + if (!account_id) { + throw Error("user must be signed in"); + } + args[0].account_id = account_id; + return args; +}; + diff --git a/src/packages/server/nats/api/index.ts b/src/packages/server/nats/api/index.ts index 572ce5e26e..50de6a6356 100644 --- a/src/packages/server/nats/api/index.ts +++ b/src/packages/server/nats/api/index.ts @@ -11,6 +11,7 @@ To do development: echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node + 3. Optional: start more servers -- requests get randomly routed to exactly one of them: echo "require('@cocalc/server/nats').default()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node @@ -106,9 +107,11 @@ import * as purchases from "./purchases"; import * as db from "./db"; import * as llm from "./llm"; import * as system from "./system"; +import * as projects from "./projects"; export const hubApi: HubApi = { system, + projects, db, llm, purchases, @@ -120,6 +123,11 @@ async function getResponse({ name, args, account_id, project_id }) { if (f == null) { throw Error(`unknown function '${name}'`); } - const args2 = transformArgs({ name, args, account_id, project_id }); + const args2 = await transformArgs({ + name, + args, + account_id, + project_id, + }); return await f(...args2); } diff --git a/src/packages/server/nats/api/projects.ts b/src/packages/server/nats/api/projects.ts new file mode 100644 index 0000000000..c54772b55c --- /dev/null +++ b/src/packages/server/nats/api/projects.ts @@ -0,0 +1,5 @@ +import { addProjectPermission } from "@cocalc/server/nats/auth"; +export { addProjectPermission }; +import createProject from "@cocalc/server/projects/create"; +export { createProject }; + diff --git a/src/packages/server/nats/api/system.ts b/src/packages/server/nats/api/system.ts index 40b062677b..7ee5d275fa 100644 --- a/src/packages/server/nats/api/system.ts +++ b/src/packages/server/nats/api/system.ts @@ -1,7 +1,5 @@ import getCustomize from "@cocalc/database/settings/customize"; export { getCustomize }; -import { addProjectPermission } from "@cocalc/server/nats/auth"; -export { addProjectPermission }; import { record_user_tracking } from "@cocalc/database/postgres/user-tracking"; import { db } from "@cocalc/database"; import manageApiKeys from "@cocalc/server/api/manage"; diff --git a/src/packages/server/projects/create.ts b/src/packages/server/projects/create.ts index 3b2a61845a..4985b0b0da 100644 --- a/src/packages/server/projects/create.ts +++ b/src/packages/server/projects/create.ts @@ -11,20 +11,13 @@ import { v4 } from "uuid"; import { associatedLicense } from "@cocalc/server/licenses/public-path"; import getFromPool from "@cocalc/server/projects/pool/get-project"; import getLogger from "@cocalc/backend/logger"; +import { getProject } from "@cocalc/server/projects/control"; +import { type CreateProjectOptions } from "@cocalc/util/db-schema/projects"; +import { delay } from "awaiting"; const log = getLogger("server:projects:create"); -interface Options { - account_id?: string; - title?: string; - description?: string; - image?: string; - license?: string; - public_path_id?: string; // may imply use of a license - noPool?: boolean; // do not allow using the pool (e.g., need this when creating projects to put in the pool); not a real issue since when creating for pool account_id is null, and then we wouldn't use the pool... -} - -export default async function createProject(opts: Options) { +export default async function createProject(opts: CreateProjectOptions) { if (opts.account_id != null) { if (!isValidUUID(opts.account_id)) { throw Error("if account_id given, it must be a valid uuid v4"); @@ -32,8 +25,15 @@ export default async function createProject(opts: Options) { } log.debug("createProject ", opts); - const { account_id, title, description, image, public_path_id, noPool } = - opts; + const { + account_id, + title, + description, + image, + public_path_id, + noPool, + start, + } = opts; let license = opts.license; if (public_path_id) { const site_license_id = await associatedLicense(public_path_id); @@ -85,7 +85,29 @@ export default async function createProject(opts: Options) { users != null ? JSON.stringify(users) : users, site_license != null ? JSON.stringify(site_license) : undefined, image ?? envs?.default ?? DEFAULT_COMPUTE_IMAGE, - ] + ], ); + + const project = getProject(project_id); + await project.state(); + if (start) { + // intentionally not blocking + startNewProject(project, project_id); + } + return project_id; } + +async function startNewProject(project, project_id: string) { + log.debug("startNewProject", { project_id }); + try { + await project.start(); + // just in case + await delay(5000); + await project.start(); + } catch (err) { + log.debug(`WARNING: problem starting new project -- ${err}`, { + project_id, + }); + } +} diff --git a/src/packages/util/db-schema/projects.ts b/src/packages/util/db-schema/projects.ts index 23b4ba8898..3eb4b27202 100644 --- a/src/packages/util/db-schema/projects.ts +++ b/src/packages/util/db-schema/projects.ts @@ -697,3 +697,19 @@ export function isExecOptsBlocking(opts: unknown): opts is ExecOptsBlocking { export type ExecOutput = ExecuteCodeOutput & { time: number; // time in ms, from user point of view. }; + +export interface CreateProjectOptions { + account_id?: string; + title?: string; + description?: string; + // (optional) image ID + image?: string; + // (optional) license id (or multiple ids separated by commas) -- if given, project will be created with this license + license?: string; + public_path_id?: string; // may imply use of a license + // noPool = do not allow using the pool (e.g., need this when creating projects to put in the pool); + // not a real issue since when creating for pool account_id is null, and then we wouldn't use the pool... + noPool?: boolean; + // start running the moment the project is created -- uses more resources, but possibly better user experience + start?: boolean; +} diff --git a/src/packages/util/message.d.ts b/src/packages/util/message.d.ts index 236a038c9a..d74c8af84b 100644 --- a/src/packages/util/message.d.ts +++ b/src/packages/util/message.d.ts @@ -34,12 +34,8 @@ export const text_file_read_from_project: any; export const write_file_to_project: any; export const write_text_file_to_project: any; export const file_written_to_project: any; -export const create_project: any; -export const project_created: any; -export const user_search: any; export const add_license_to_project: any; export const remove_license_from_project: any; -export const user_search_results: any; export const project_users: any; export const invite_collaborator: any; export const add_collaborator: any; diff --git a/src/packages/util/message.js b/src/packages/util/message.js index 305cffe42d..dcbdc963b4 100644 --- a/src/packages/util/message.js +++ b/src/packages/util/message.js @@ -962,148 +962,8 @@ message({ // Managing multiple projects //########################################### -// client --> hub -API( - message2({ - event: "create_project", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - title: { - init: "", - desc: "project title", - }, - description: { - init: "", - desc: "project description", - }, - image: { - init: undefined, - desc: "(optional) image ID", - }, - license: { - init: undefined, - desc: "(optional) license id (or multiple ids separated by commas) -- if given, project will be created with this license", - }, - start: { - init: false, - desc: "start running the moment the project is created -- uses more resources, but possibly better user experience", - }, - noPool: { - init: false, - desc: "if true, never get project from pool, e.g., useful when creating hundreds of projects for students in a class, since they aren't immediately going to use their project", - }, - }, - desc: `\ -Example: -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d title='MY NEW PROJECT' \\ - -d description='sample project' \\ - https://cocalc.com/api/v1/create_project - == > {"event":"project_created", - "id":"0b4df293-d518-45d0-8a3c-4281e501b85e", - "project_id":"07897899-6bbb-4fbc-80a7-3586c43348d1"} -\`\`\`\ -`, - }), -); - -// hub --> client -message({ - event: "project_created", - id: required, - project_id: required, -}); - //# search --------------------------- -// client --> hub -API( - message2({ - event: "user_search", - fields: { - id: { - init: undefined, - desc: "A unique UUID for the query", - }, - query: { - init: required, - desc: "comma separated list of email addresses or strings such as 'foo bar'", - }, - admin: { - init: false, - desc: "if true and user is an admin, includes email addresses in result, and does more permissive search", - }, - active: { - init: "", - desc: "only include users active for this interval of time", - }, - limit: { - init: 20, - desc: "maximum number of results returned", - }, - only_email: { - init: false, - desc: "only search using email addresses" - }, - }, - desc: `\ - - -Examples: - -Search for account by email. -\`\`\` - curl -u : \\ - -d query=jd@m.local \\ - https://cocalc.com/api/v1/user_search - ==> {"event":"user_search_results", - "id":"3818fa50-b892-4167-b9d9-d22d521b36af", - "results":[{"account_id":"96c523b8-321e-41a3-9523-39fde95dc71d", - "first_name":"John", - "last_name":"Doe", - "email_address":"jd@m.local"} -\`\`\` - -Search for at most 3 accounts where first and last name begin with 'foo' or 'bar'. -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d 'query=foo bar'\\ - -d limit=3 \\ - https://cocalc.com/api/v1/user_search - ==> {"event":"user_search_results", - "id":"fd9b025b-25d0-4e27-97f4-2c080bb07155", - "results":[{"account_id":"1a842a67-eed3-405d-a222-2f23a33f675e", - "first_name":"foo", - "last_name":"bar"}, - {"account_id":"0e9418a7-af6a-4004-970a-32fafe733f29", - "first_name":"bar123", - "last_name":"fooxyz"}, - {"account_id":"93f8131c-6c21-401a-897d-d4abd9c6c225", - "first_name":"Foo", - "last_name":"Bar"}]} -\`\`\` - -The same result as the last example above would be returned with a -search string of 'bar foo'. -A name of "Xfoo YBar" would not match. - -Note that email addresses are not returned for string search items. - -Email and string search types may be mixed in a single query: -\`\`\` - curl -u sk_abcdefQWERTY090900000000: \\ - -d 'query=foo bar,jd@m.local' \\ - -d limit=4 \\ - https://cocalc.com/api/v1/user_search -\`\`\`\ -`, - }), -); - API( message2({ event: "add_license_to_project", From 8c04d873a60183b0bca5c45d463771c1a9af9c85 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 17:24:23 +0000 Subject: [PATCH 117/281] nats: add new api support for read/write text file to project --- src/packages/frontend/client/project.ts | 31 ++++++++----- src/packages/hub/api/examples.md | 11 ----- src/packages/hub/client.coffee | 58 ------------------------- src/packages/nats/project-api/system.ts | 10 ++++- src/packages/project/nats/api/system.ts | 22 ++++++++++ src/packages/util/message.d.ts | 1 - 6 files changed, 51 insertions(+), 82 deletions(-) delete mode 100644 src/packages/hub/api/examples.md diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index 01bbf0c517..9182636ec2 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -69,19 +69,35 @@ export class ProjectClient { return await this.client.async_call({ message }); } - public async write_text_file(opts: { + private natsApi = (project_id: string) => { + return this.client.nats_client.projectApi({ project_id }); + }; + + public async write_text_file({ + project_id, + path, + content, + }: { project_id: string; path: string; content: string; }): Promise { - return await this.call(message.write_text_file_to_project(opts)); + await this.natsApi(project_id).system.writeTextFileToProject({ + path, + content, + }); } - public async read_text_file(opts: { + public async read_text_file({ + project_id, + path, + }: { project_id: string; // string or array of strings path: string; // string or array of strings }): Promise { - return (await this.call(message.read_text_file_from_project(opts))).content; + return await this.natsApi(project_id).system.readTextFileFromProject({ + path, + }); } // Like "read_text_file" above, except the callback @@ -319,13 +335,6 @@ export class ProjectClient { return { files: listing }; } - public async public_get_text_file(opts: { - project_id: string; - path: string; - }): Promise { - return (await this.call(message.public_get_text_file(opts))).data; - } - public async find_directories(opts: { project_id: string; query?: string; // see the -iwholename option to the UNIX find command. diff --git a/src/packages/hub/api/examples.md b/src/packages/hub/api/examples.md deleted file mode 100644 index 95b2c81c2c..0000000000 --- a/src/packages/hub/api/examples.md +++ /dev/null @@ -1,11 +0,0 @@ -curl -X POST -u sk_BS9fVpiEpPpJbhSZurAMSnmM: http://localhost:50195/api/v1/ping - -curl -X POST -d title='API project' -d description='Stuff' -u sk_BS9fVpiEpPpJbhSZurAMSnmM: http://localhost:50195/api/v1/create_project - -curl -d first_name=API -d last_name=gal -d email_address=a@b -d password=666 -d agreed_to_terms=true http://localhost:54249/api/v1/create_account - -curl -d command="ls -al" -d project_id="72b622c0-665d-4512-8bc6-197ecdba1d8b" http://localhost:54249/api/v1/project_exec - -curl -d path='a.txt' -d project_id="72b622c0-665d-4512-8bc6-197ecdba1d8b" http://localhost:54249/api/v1/read_text_file_from_project - -curl -d content='foo bar' -d path='a.txt' -d project_id="72b622c0-665d-4512-8bc6-197ecdba1d8b" http://localhost:54249/api/v1/write_text_file_to_project \ No newline at end of file diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee index 421b0a896b..60dad8098a 100644 --- a/src/packages/hub/client.coffee +++ b/src/packages/hub/client.coffee @@ -755,64 +755,6 @@ class exports.Client extends EventEmitter cb(undefined, project) ) - mesg_write_text_file_to_project: (mesg) => - @get_project mesg, 'write', (err, project) => - if err - return - project.write_file - path : mesg.path - data : mesg.content - cb : (err) => - if err - @error_to_client(id:mesg.id, error:err) - else - @push_to_client(message.file_written_to_project(id:mesg.id)) - - mesg_read_text_files_from_projects: (mesg) => - if not misc.is_array(mesg.project_id) - @error_to_client(id:mesg.id, error:"project_id must be an array") - return - if not misc.is_array(mesg.path) or mesg.path.length != mesg.project_id.length - @error_to_client(id:mesg.id, error:"if project_id is an array, then path must be an array of the same length") - return - v = [] - f = (mesg, cb) => - @get_project mesg, 'read', (err, project) => - if err - cb(err) - return - project.read_file - path : mesg.path - cb : (err, content) => - if err - v.push({path:mesg.path, project_id:mesg.project_id, error:err}) - else - v.push({path:mesg.path, project_id:mesg.project_id, content:content.blob.toString()}) - cb() - paths = [] - for i in [0...mesg.project_id.length] - paths.push({id:mesg.id, path:mesg.path[i], project_id:mesg.project_id[i]}) - async.mapLimit paths, 20, f, (err) => - if err - @error_to_client(id:mesg.id, error:err) - else - @push_to_client(message.text_file_read_from_project(id:mesg.id, content:v)) - - mesg_read_text_file_from_project: (mesg) => - if misc.is_array(mesg.project_id) - @mesg_read_text_files_from_projects(mesg) - return - @get_project mesg, 'read', (err, project) => - if err - return - project.read_file - path : mesg.path - cb : (err, content) => - if err - @error_to_client(id:mesg.id, error:err) - else - t = content.blob.toString() - @push_to_client(message.text_file_read_from_project(id:mesg.id, content:t)) mesg_project_exec: (mesg) => if mesg.command == "ipython-notebook" diff --git a/src/packages/nats/project-api/system.ts b/src/packages/nats/project-api/system.ts index a7b6d9f685..61c5c63f09 100644 --- a/src/packages/nats/project-api/system.ts +++ b/src/packages/nats/project-api/system.ts @@ -20,6 +20,9 @@ export const system = { realpath: true, canonicalPaths: true, + writeTextFileToProject: true, + readTextFileFromProject: true, + configuration: true, ping: true, @@ -41,6 +44,12 @@ export interface System { realpath: (path: string) => Promise; canonicalPaths: (paths: string[]) => Promise; + writeTextFileToProject: (opts: { + path: string; + content: string; + }) => Promise; + readTextFileFromProject: (opts: { path: string }) => Promise; + configuration: ( aspect: ConfigurationAspect, no_cache?, @@ -49,5 +58,4 @@ export interface System { ping: () => Promise<{ now: number }>; exec: (opts: ExecuteCodeOptions) => Promise; - } diff --git a/src/packages/project/nats/api/system.ts b/src/packages/project/nats/api/system.ts index 0321e2b2cf..e6a808b217 100644 --- a/src/packages/project/nats/api/system.ts +++ b/src/packages/project/nats/api/system.ts @@ -52,3 +52,25 @@ export { get_configuration as configuration }; import { canonical_paths } from "../../browser-websocket/canonical-path"; export { canonical_paths as canonicalPaths }; + +import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; +import { readFile, writeFile } from "fs/promises"; + +export async function writeTextFileToProject({ + path, + content, +}: { + path: string; + content: string; +}): Promise { + await ensureContainingDirectoryExists(path); + await writeFile(path, content); +} + +export async function readTextFileFromProject({ + path, +}: { + path: string; +}): Promise { + return (await readFile(path)).toString(); +} diff --git a/src/packages/util/message.d.ts b/src/packages/util/message.d.ts index d74c8af84b..1edb1e30dd 100644 --- a/src/packages/util/message.d.ts +++ b/src/packages/util/message.d.ts @@ -121,5 +121,4 @@ export const jupyter_kernels: any; export const jupyter_start_pool: any; export const forgot_password: any; export const reset_forgot_password: any; -export const public_get_text_file: any; export const signal_sent: any; From c90e4e32a529de5204ab709b2a528ce4388ed1a2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 21:58:35 +0000 Subject: [PATCH 118/281] nats: improve the kv a little more --- src/packages/nats/sync/kv.ts | 144 +++++++++++++++++++++++++++++++---- src/packages/nats/util.ts | 22 ++++++ 2 files changed, 151 insertions(+), 15 deletions(-) diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index 6b8a1ada5d..c307934a58 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -22,12 +22,54 @@ This is a simple KV wrapper around NATS's KV, for small KV stores, suitable for faster for bigger objects, but means that if "await set({...})" fails, you don't immediately know which keys were successfully set and which failed, though all that worked will get updated soon and reflected in get(). + +TODO: + +- [ ] maybe expose some functionality related to versions/history? + + +DEVELOPMENT: + +~/cocalc/src/packages/server$ n +Welcome to Node.js v18.17.1. +Type ".help" for more information. +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/kv"); s = new a.KV({name:'test',env,subjects:['foo.>']}); await s.init(); + +> await s.set({"foo.x":10}) // or s.set("foo.x", 10) +> s.get() +{ 'foo.x': 10 } +> await s.delete("foo.x") +undefined +> s.get() +{} +> await s.set({"foo.x":10, "foo.bar":20}) + +// Since the subjects are disjoint these are totally different: + +> t = new a.KV({name:'test',env,subjects:['bar.>']}); await t.init(); +> await t.get() +{} +> await t.set({"bar.abc":10}) +undefined +> await t.get() +{ 'bar.abc': Uint8Array(2) [ 49, 48 ] } +> await s.get() +{ 'foo.x': 10, 'foo.bar': 20, 'bar.abc': 10 } + +// The union: +> u = new a.KV({name:'test',env,subjects:['bar.>', 'foo.>']}); await u.init(); +> u.get() +{ 'foo.x': 10, 'foo.bar': 20, 'bar.abc': 10 } +> await s.set({'foo.x':999}) +undefined +> u.get() +{ 'foo.x': 999, 'foo.bar': 20, 'bar.abc': 10 } */ import { EventEmitter } from "events"; import { type NatsEnv } from "@cocalc/nats/types"; import { Kvm } from "@nats-io/kv"; -import { getAllFromKv } from "@cocalc/nats/util"; +import { getAllFromKv, matchesPattern } from "@cocalc/nats/util"; import { isEqual } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { map as awaitMap } from "awaiting"; @@ -37,6 +79,7 @@ const MAX_PARALLEL_SET = 50; export class KV extends EventEmitter { public readonly name: string; private options?; + private subjects?: string | string[]; private env: NatsEnv; private kv?; private watch?; @@ -47,8 +90,12 @@ export class KV extends EventEmitter { name, env, options, + subjects, }: { name: string; + // optionally restrict to subset of named kv store matching these subjects. + // NOTE: any key name that you *set or delete* should match one of these + subjects?: string | string[]; env: NatsEnv; options?; }) { @@ -56,6 +103,16 @@ export class KV extends EventEmitter { this.env = env; this.name = name; this.options = options; + this.subjects = subjects; + return new Proxy(this, { + set(target, prop, value) { + target.setOne(prop, value); + return true; + }, + get(target, prop) { + return target[prop] ?? target.all?.[String(prop)]; + }, + }); } init = reuseInFlight(async () => { @@ -69,10 +126,11 @@ export class KV extends EventEmitter { }); const { all, revisions } = await getAllFromKv({ kv: this.kv, + key: this.subjects, }); this.revisions = revisions; - for (const k in all) { - all[k] = this.env.jc.decode(all[k]); + for (const key in all) { + all[key] = this.env.jc.decode(all[key]); } this.all = all; this.emit("connected"); @@ -110,22 +168,63 @@ export class KV extends EventEmitter { this.removeAllListeners(); }; - get = () => { - return { ...this.all }; + get = (key?) => { + if (key == undefined) { + return { ...this.all }; + } else { + return this.all?.[key]; + } + }; + + private matches = (key: string) => { + if (this.subjects == null) { + return true; + } + for (const pattern of this.subjects) { + if (matchesPattern({ pattern, subject: key })) { + return true; + } + } + return false; }; delete = async (key) => { + if (!this.matches(key)) { + throw Error( + `delete: key (=${key}) must match one of the subjects: ${JSON.stringify(this.subjects)}`, + ); + } if (this.all == null || this.revisions == null) { throw Error("not ready"); } - if (this.all[key] != null) { - const newRevision = await this.kv.delete(key); - this.revisions[key] = newRevision; + if (this.all[key] !== undefined) { + const cur = this.all[key]; + try { + delete this.all[key]; + const newRevision = await this.kv.delete(key); + this.revisions[key] = newRevision; + } catch (err) { + if (cur === undefined) { + delete this.all[key]; + } else { + this.all[key] = cur; + } + throw err; + } } - delete this.all[key]; }; - set = async (obj) => { + // delete all that we know about + clear = async () => { + await awaitMap(Object.keys(this.all), MAX_PARALLEL_SET, this.delete); + }; + + set = async (...args) => { + if (args.length == 2) { + await this.setOne(args[0], args[1]); + return; + } + const obj = args[0]; await awaitMap( Object.keys(obj), MAX_PARALLEL_SET, @@ -134,6 +233,11 @@ export class KV extends EventEmitter { }; private setOne = async (key, value) => { + if (!this.matches(key)) { + throw Error( + `set: key (=${key}) must match one of the subjects: ${JSON.stringify(this.subjects)}`, + ); + } if (this.all == null || this.revisions == null) { throw Error("not ready"); } @@ -145,10 +249,20 @@ export class KV extends EventEmitter { } const revision = this.revisions[key]; const val = this.env.jc.encode(value); - const newRevision = await this.kv.put(key, val, { - previousSeq: revision, - }); - this.revisions[key] = newRevision; - this.all[key] = val; + const cur = this.all[key]; + try { + this.all[key] = value; + const newRevision = await this.kv.put(key, val, { + previousSeq: revision, + }); + this.revisions[key] = newRevision; + } catch (err) { + if (cur === undefined) { + delete this.all[key]; + } else { + this.all[key] = cur; + } + throw err; + } }; } diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index 5d8d18d319..6e563b4b9c 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -73,3 +73,25 @@ export function handleErrorMessage(mesg) { } return mesg; } + +// Returns true if the subject matches the NATS pattern. +export function matchesPattern({ + pattern, + subject, +}: { + pattern: string; + subject: string; +}): boolean { + const subParts = subject.split("."); + const patParts = pattern.split("."); + let i = 0, + j = 0; + while (i < subParts.length && j < patParts.length) { + if (patParts[j] === ">") return true; + if (patParts[j] !== "*" && patParts[j] !== subParts[i]) return false; + i++; + j++; + } + + return i === subParts.length && j === patParts.length; +} From 108f3fe80b63fc16d087afc8f86d7a05423eb201 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 4 Feb 2025 23:18:23 +0000 Subject: [PATCH 119/281] nats: improve kv store even more --- src/packages/frontend/client/nats.ts | 30 ++++++++ src/packages/nats/sync/kv.ts | 100 +++++++++++++++++++++------ src/packages/nats/sync/open-files.ts | 2 +- src/packages/nats/util.ts | 9 ++- src/packages/server/nats/auth.ts | 4 +- 5 files changed, 117 insertions(+), 28 deletions(-) diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/client/nats.ts index 5ff5625275..fb2da1bef3 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/client/nats.ts @@ -18,6 +18,7 @@ import { OpenFiles } from "@cocalc/nats/sync/open-files"; import { PubSub } from "@cocalc/nats/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; import { SystemKv } from "@cocalc/nats/system"; +import { KV } from "@cocalc/nats/sync/kv"; export class NatsClient { /*private*/ client: WebappClient; @@ -323,4 +324,33 @@ export class NatsClient { } return this.theSystemKv!; }); + + private kvCache: { [key: string]: KV } = {}; + kv = reuseInFlight( + async ({ + name, + filter, + options, + }: { + name: string; + filter?: string | string[]; + options?; + }) => { + const key = JSON.stringify([name, filter, options]); + if (this.kvCache[key] == null) { + const kv = new KV({ + name, + filter, + options, + env: await this.getEnv(), + }); + await kv.init(); + this.kvCache[key] = kv; + kv.on("closed", () => { + delete this.kvCache[key]; + }); + } + return this.kvCache[key]; + }, + ); } diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index c307934a58..dbabe6e98a 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -17,7 +17,7 @@ This is a simple KV wrapper around NATS's KV, for small KV stores, suitable for is in the local cache on multiple nodes at once anywhere, and be 100% certain to never overwrite data in complicated objects. Of course, you have to assume "await set()" will sometimes fail. - - set "pipelines" in that MAX_PARALLEL_SET key/value pairs are set at once, without waiting + - set "pipelines" in that MAX_PARALLEL key/value pairs are set at once, without waiting for each set to get ACK'd from the server before doing more sets. This makes this massively faster for bigger objects, but means that if "await set({...})" fails, you don't immediately know which keys were successfully set and which failed, though all that worked will get @@ -33,7 +33,7 @@ DEVELOPMENT: ~/cocalc/src/packages/server$ n Welcome to Node.js v18.17.1. Type ".help" for more information. -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/kv"); s = new a.KV({name:'test',env,subjects:['foo.>']}); await s.init(); +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/kv"); s = new a.KV({name:'test',env,filter:['foo.>']}); await s.init(); > await s.set({"foo.x":10}) // or s.set("foo.x", 10) > s.get() @@ -44,9 +44,9 @@ undefined {} > await s.set({"foo.x":10, "foo.bar":20}) -// Since the subjects are disjoint these are totally different: +// Since the filters are disjoint these are totally different: -> t = new a.KV({name:'test',env,subjects:['bar.>']}); await t.init(); +> t = new a.KV({name:'test',env,filter:['bar.>']}); await t.init(); > await t.get() {} > await t.set({"bar.abc":10}) @@ -57,7 +57,7 @@ undefined { 'foo.x': 10, 'foo.bar': 20, 'bar.abc': 10 } // The union: -> u = new a.KV({name:'test',env,subjects:['bar.>', 'foo.>']}); await u.init(); +> u = new a.KV({name:'test',env,filter:['bar.>', 'foo.>']}); await u.init(); > u.get() { 'foo.x': 10, 'foo.bar': 20, 'bar.abc': 10 } > await s.set({'foo.x':999}) @@ -74,28 +74,29 @@ import { isEqual } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { map as awaitMap } from "awaiting"; -const MAX_PARALLEL_SET = 50; +const MAX_PARALLEL = 50; export class KV extends EventEmitter { public readonly name: string; private options?; - private subjects?: string | string[]; + private filter?: string[]; private env: NatsEnv; private kv?; private watch?; private all?: { [key: string]: any }; private revisions?: { [key: string]: number }; + private times?: { [key: string]: Date }; constructor({ name, env, + filter, options, - subjects, }: { name: string; - // optionally restrict to subset of named kv store matching these subjects. + // filter: optionally restrict to subset of named kv store matching these subjects. // NOTE: any key name that you *set or delete* should match one of these - subjects?: string | string[]; + filter?: string | string[]; env: NatsEnv; options?; }) { @@ -103,7 +104,12 @@ export class KV extends EventEmitter { this.env = env; this.name = name; this.options = options; - this.subjects = subjects; + this.filter = + filter == null + ? undefined + : typeof filter == "string" + ? [filter] + : filter; return new Proxy(this, { set(target, prop, value) { target.setOne(prop, value); @@ -124,11 +130,12 @@ export class KV extends EventEmitter { compression: true, ...this.options, }); - const { all, revisions } = await getAllFromKv({ + const { all, revisions, times } = await getAllFromKv({ kv: this.kv, - key: this.subjects, + key: this.filter, }); this.revisions = revisions; + this.times = times; for (const key in all) { all[key] = this.env.jc.decode(all[key]); } @@ -143,9 +150,11 @@ export class KV extends EventEmitter { // we assume that we ONLY delete old items which are not relevant ignoreDeletes: true, include: "updates", + key: this.filter, }); //for await (const { key, value } of this.watch) { - for await (const { revision, key, value } of this.watch) { + for await (const x of this.watch) { + const { revision, key, value, sm } = x; if (this.revisions == null || this.all == null) { return; } @@ -153,8 +162,10 @@ export class KV extends EventEmitter { if (value.length == 0) { // delete delete this.all[key]; + delete this.times[key]; } else { this.all[key] = this.env.jc.decode(value); + this.times[key] = sm.time; } this.emit("change", key, this.all[key]); } @@ -163,12 +174,16 @@ export class KV extends EventEmitter { close = () => { this.watch?.stop(); delete this.all; + delete this.times; delete this.revisions; this.emit("closed"); this.removeAllListeners(); }; get = (key?) => { + if (this.all == null) { + throw Error("not initialized"); + } if (key == undefined) { return { ...this.all }; } else { @@ -176,11 +191,18 @@ export class KV extends EventEmitter { } }; + time = (key?) => { + if (key == null) { + return this.times; + } else { + return this.times?.[key]; + } + }; private matches = (key: string) => { - if (this.subjects == null) { + if (this.filter == null) { return true; } - for (const pattern of this.subjects) { + for (const pattern of this.filter) { if (matchesPattern({ pattern, subject: key })) { return true; } @@ -188,10 +210,10 @@ export class KV extends EventEmitter { return false; }; - delete = async (key) => { + delete = async (key, revision?) => { if (!this.matches(key)) { throw Error( - `delete: key (=${key}) must match one of the subjects: ${JSON.stringify(this.subjects)}`, + `delete: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, ); } if (this.all == null || this.revisions == null) { @@ -201,8 +223,11 @@ export class KV extends EventEmitter { const cur = this.all[key]; try { delete this.all[key]; - const newRevision = await this.kv.delete(key); + const newRevision = await this.kv.delete(key, { + previousSeq: revision ?? this.revisions[key], + }); this.revisions[key] = newRevision; + delete this.times[key]; } catch (err) { if (cur === undefined) { delete this.all[key]; @@ -214,9 +239,40 @@ export class KV extends EventEmitter { } }; + // delete everything matching the filter that hasn't been set + // in the given amount of ms. Returns number of deleted records. + // NOTE: This could throw an exception if something that would expire + // were changed right when this is run then it would get expired + // but shouldn't. In that case, run it again. + expire = async (ageMs: number): Promise => { + if (!ageMs) { + throw Error("ageMs must be set"); + } + if (this.times == null || this.all == null) { + throw Error("not initialized"); + } + const cutoff = new Date(Date.now() - ageMs); + // make copy of revisions *before* we start deleting so that + // if a key is changed exactly while deleting we get an error + // and don't accidently delete it! + const revisions = { ...this.revisions }; + const toDelete = Object.keys(this.all).filter( + (key) => this.times?.[key] != null && this.times[key] <= cutoff, + ); + if (toDelete.length > 0) { + await awaitMap(toDelete, MAX_PARALLEL, async (key) => { + await this.delete(key, revisions[key]); + }); + } + return toDelete.length; + }; + // delete all that we know about clear = async () => { - await awaitMap(Object.keys(this.all), MAX_PARALLEL_SET, this.delete); + if (this.all == null) { + throw Error("not initialized"); + } + await awaitMap(Object.keys(this.all), MAX_PARALLEL, this.delete); }; set = async (...args) => { @@ -227,7 +283,7 @@ export class KV extends EventEmitter { const obj = args[0]; await awaitMap( Object.keys(obj), - MAX_PARALLEL_SET, + MAX_PARALLEL, async (key) => await this.setOne(key, obj[key]), ); }; @@ -235,7 +291,7 @@ export class KV extends EventEmitter { private setOne = async (key, value) => { if (!this.matches(key)) { throw Error( - `set: key (=${key}) must match one of the subjects: ${JSON.stringify(this.subjects)}`, + `set: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, ); } if (this.all == null || this.revisions == null) { diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 2d7963ed41..9288e2cfa0 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -171,7 +171,7 @@ export class OpenFiles { // delete entries that haven't been touched in ageMs milliseconds. // default=a month // returns number of deleted objects. - deleteOld = async (ageMs: number = 1000 * 60 * 60 * 730): Promise => { + expire = async (ageMs: number = 1000 * 60 * 60 * 730): Promise => { let n = 0; const cutoff = new Date(Date.now() - ageMs); const kv = await this.getKv(); diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index 6e563b4b9c..3369112620 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -27,19 +27,22 @@ export async function getAllFromKv({ }): Promise<{ all: { [key: string]: any }; revisions: { [key: string]: number }; + times: { [key: string]: Date }; }> { const total = await numKeys(kv, key); let count = 0; const all: any = {}; const revisions: { [key: string]: number } = {}; + const times: { [key: string]: Date } = {}; if (total == 0) { - return { all, revisions }; + return { all, revisions, times }; } const watch = await kv.watch({ key, ignoreDeletes: true }); let id: any = 0; - for await (const { key, value, revision } of watch) { + for await (const { key, value, revision, sm } of watch) { all[key] = value; revisions[key] = revision; + times[key] = sm.time; count += 1; @@ -60,7 +63,7 @@ export async function getAllFromKv({ watch.stop(); }, timeout); } - return { all, revisions }; + return { all, revisions, times }; } export function handleErrorMessage(mesg) { diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index b9dd410b7a..7d369944a4 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -18,8 +18,7 @@ DOCS: USAGE: -a = require('@cocalc/server/nats/auth'); -await a.configureNatsUser({account_id:'275f1db7-bf37-4b44-b9aa-d64694269c9f'}) +a = require('@cocalc/server/nats/auth'); await a.configureNatsUser({account_id:'275f1db7-bf37-4b44-b9aa-d64694269c9f'}) await a.configureNatsUser({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}) */ @@ -124,6 +123,7 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { if (userType == "account") { goalSub.add(`*.account-${userId}.>`); + goalPub.add(`*.account-${userId}.>`); const pool = getPool(); // all RUNNING projects with the user's group From 6ff85e570963f5c4f66ab08d5dc71a4869a2d993 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 5 Feb 2025 00:29:25 +0000 Subject: [PATCH 120/281] fix some typescript issue with kv --- src/packages/nats/sync/kv.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index dbabe6e98a..6b154870a4 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -155,7 +155,7 @@ export class KV extends EventEmitter { //for await (const { key, value } of this.watch) { for await (const x of this.watch) { const { revision, key, value, sm } = x; - if (this.revisions == null || this.all == null) { + if (this.revisions == null || this.all == null || this.times == null) { return; } this.revisions[key] = revision; @@ -216,7 +216,7 @@ export class KV extends EventEmitter { `delete: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, ); } - if (this.all == null || this.revisions == null) { + if (this.all == null || this.revisions == null || this.times == null) { throw Error("not ready"); } if (this.all[key] !== undefined) { From 10390faac6e66f088e19c89f0a2d968b8ba84b04 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 5 Feb 2025 00:58:43 +0000 Subject: [PATCH 121/281] nats: work in progress rewriting copy path between projects... --- src/packages/nats/hub-api/projects.ts | 3 ++ src/packages/server/nats/api/projects.ts | 39 ++++++++++++++++++++ src/packages/server/projects/control/base.ts | 22 +++-------- src/packages/util/db-schema/projects.ts | 26 +++++++++++++ 4 files changed, 74 insertions(+), 16 deletions(-) diff --git a/src/packages/nats/hub-api/projects.ts b/src/packages/nats/hub-api/projects.ts index 13b892be6d..e60b58798e 100644 --- a/src/packages/nats/hub-api/projects.ts +++ b/src/packages/nats/hub-api/projects.ts @@ -1,13 +1,16 @@ import { authFirstRequireAccount } from "./util"; import { type CreateProjectOptions } from "@cocalc/util/db-schema/projects"; +import { type UserCopyOptions } from "@cocalc/util/db-schema/projects"; export const projects = { addProjectPermission: authFirstRequireAccount, createProject: authFirstRequireAccount, + copyPathBetweenProjects: authFirstRequireAccount, }; export interface Projects { // request to have NATS permissions to project subjects. addProjectPermission: (opts: { project_id: string }) => Promise; createProject: (opts: CreateProjectOptions) => Promise; + copyPathBetweenProjects: (opts: UserCopyOptions) => Promise; } diff --git a/src/packages/server/nats/api/projects.ts b/src/packages/server/nats/api/projects.ts index c54772b55c..4b2c7d8dca 100644 --- a/src/packages/server/nats/api/projects.ts +++ b/src/packages/server/nats/api/projects.ts @@ -3,3 +3,42 @@ export { addProjectPermission }; import createProject from "@cocalc/server/projects/create"; export { createProject }; +import { type UserCopyOptions } from "@cocalc/util/db-schema/projects"; +import { getProject } from "@cocalc/server/projects/control"; +import isCollaborator from "@cocalc/server/projects/is-collaborator"; +import { delay } from "awaiting"; + +export async function copyPathBetweenProjects( + opts: UserCopyOptions, +): Promise { + const { account_id, src_project_id, target_project_id } = opts; + if (!account_id) { + throw Error("user must be signed in"); + } + if (!(await isCollaborator({ account_id, project_id: src_project_id }))) { + throw Error("user must be collaborator on source project"); + } + if ( + !!target_project_id && + target_project_id != src_project_id && + !(await isCollaborator({ account_id, project_id: target_project_id })) + ) { + throw Error("user must be collaborator on target project"); + } + + await doCopyPathBetweenProjects(opts); +} + +// do the actual copy, awaiting as long as it takes to finish, +// with no security checks. +async function doCopyPathBetweenProjects(opts: UserCopyOptions) { + const project = await getProject(opts.src_project_id); + await project.copyPath({ + ...opts, + path: opts.src_path, + wait_until_done: true, + }); + if (opts.debug_delay_ms) { + await delay(opts.debug_delay_ms); + } +} diff --git a/src/packages/server/projects/control/base.ts b/src/packages/server/projects/control/base.ts index 01bb1e4e60..606b12daa5 100644 --- a/src/packages/server/projects/control/base.ts +++ b/src/packages/server/projects/control/base.ts @@ -25,7 +25,11 @@ import { callback2 } from "@cocalc/util/async-utils"; import { db } from "@cocalc/database"; import { EventEmitter } from "events"; import { isEqual } from "lodash"; -import { ProjectState, ProjectStatus } from "@cocalc/util/db-schema/projects"; +import { + type CopyOptions, + ProjectState, + ProjectStatus, +} from "@cocalc/util/db-schema/projects"; import { Quota, quota } from "@cocalc/util/upgrades/quota"; import { delay } from "awaiting"; import getLogger from "@cocalc/backend/logger"; @@ -36,6 +40,7 @@ import { closePayAsYouGoPurchases } from "@cocalc/server/purchases/project-quota import { handlePayAsYouGoQuotas } from "./pay-as-you-go"; import { query } from "@cocalc/database/postgres/query"; +export type { CopyOptions }; export type { ProjectState, ProjectStatus }; const logger = getLogger("project-control"); @@ -283,18 +288,3 @@ export abstract class BaseProject extends EventEmitter { logger.debug("updated run_quota=", run_quota); } } - -export interface CopyOptions { - path: string; - target_project_id?: string; - target_path?: string; // path into project; if not given, defaults to source path above. - overwrite_newer?: boolean; // if true, newer files in target are copied over (otherwise, uses rsync's --update) - delete_missing?: boolean; // if true, delete files in dest path not in source, **including** newer files - backup?: boolean; // make backup files - timeout?: number; // in **seconds**, not milliseconds - bwlimit?: number; - wait_until_done?: boolean; // by default, wait until done. false only gives the ID to query the status later - scheduled?: string | Date; // kucalc only: string (parseable by new Date()), or a Date - public?: boolean; // kucalc only: if true, may use the share server files rather than start the source project running - exclude?: string[]; // options passed to rsync via --exclude -} diff --git a/src/packages/util/db-schema/projects.ts b/src/packages/util/db-schema/projects.ts index 3eb4b27202..35f861fed1 100644 --- a/src/packages/util/db-schema/projects.ts +++ b/src/packages/util/db-schema/projects.ts @@ -713,3 +713,29 @@ export interface CreateProjectOptions { // start running the moment the project is created -- uses more resources, but possibly better user experience start?: boolean; } + +interface BaseCopyOptions { + target_project_id?: string; + target_path?: string; // path into project; if not given, defaults to source path above. + overwrite_newer?: boolean; // if true, newer files in target are copied over (otherwise, uses rsync's --update) + delete_missing?: boolean; // if true, delete files in dest path not in source, **including** newer files + backup?: boolean; // make backup files + timeout?: number; // in **seconds**, not milliseconds + bwlimit?: number; + wait_until_done?: boolean; // by default, wait until done. false only gives the ID to query the status later + scheduled?: string | Date; // kucalc only: string (parseable by new Date()), or a Date + public?: boolean; // kucalc only: if true, may use the share server files rather than start the source project running + exclude?: string[]; // options passed to rsync via --exclude +} +export interface UserCopyOptions extends BaseCopyOptions { + account_id?: string; + src_project_id: string; + src_path: string; + // simulate copy taking at least this long -- useful for dev/debugging. + debug_delay_ms?: number; +} + +// for copying files within and between projects +export interface CopyOptions extends BaseCopyOptions { + path: string; +} From e0a4c0047f00fdda0967423e8a17835bdb6ad454 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 5 Feb 2025 05:04:20 +0000 Subject: [PATCH 122/281] nats: first attempt at simple browser api. - works but there's a bunch of timeout errors initially on startup --- src/packages/frontend/client/client.ts | 2 +- src/packages/frontend/nats/api/index.ts | 62 +++++++++++++++++ src/packages/frontend/nats/api/system.ts | 10 +++ .../{client/nats.ts => nats/client.ts} | 66 ++++++++++++++++++- src/packages/nats/browser-api/index.ts | 48 ++++++++++++++ src/packages/nats/browser-api/system.ts | 9 +++ src/packages/nats/names.ts | 6 +- src/packages/nats/package.json | 1 + 8 files changed, 198 insertions(+), 6 deletions(-) create mode 100644 src/packages/frontend/nats/api/index.ts create mode 100644 src/packages/frontend/nats/api/system.ts rename src/packages/frontend/{client/nats.ts => nats/client.ts} (86%) create mode 100644 src/packages/nats/browser-api/index.ts create mode 100644 src/packages/nats/browser-api/system.ts diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 4428815a4b..cabfff8404 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -21,7 +21,7 @@ import { SyncClient } from "@cocalc/sync/client/sync-client"; import { UsersClient } from "./users"; import { FileClient } from "./file"; import { TrackingClient } from "./tracking"; -import { NatsClient } from "./nats"; +import { NatsClient } from "@cocalc/frontend/nats/client"; import { HubClient } from "./hub"; import { IdleClient } from "./idle"; import { version } from "@cocalc/util/smc-version"; diff --git a/src/packages/frontend/nats/api/index.ts b/src/packages/frontend/nats/api/index.ts new file mode 100644 index 0000000000..aef7172de6 --- /dev/null +++ b/src/packages/frontend/nats/api/index.ts @@ -0,0 +1,62 @@ +/* + +*/ + +import { webapp_client } from "@cocalc/frontend/webapp-client"; +import { type BrowserApi } from "@cocalc/nats/browser-api"; +import { browserSubject } from "@cocalc/nats/names"; +import { delay } from "awaiting"; + +export async function initApi() { + console.log("init nats browser api - x"); + const sessionId = webapp_client.nats_client.sessionId; + console.log({ sessionId }); + // TODO: neeed to wait for signed in event! + while (!webapp_client.account_id) { + await delay(200); + } + const subject = browserSubject({ + account_id: webapp_client.account_id, + sessionId, + service: "api", + }); + console.log({ subject }); + console.log("init browser API", { sessionId, subject }); + const { jc, nc } = await webapp_client.nats_client.getEnv(); + const subscription = nc.subscribe(subject); + listen({ subscription, jc }); +} + +async function listen({ subscription, jc }) { + for await (const mesg of subscription) { + const request = jc.decode(mesg.data); + handleApiRequest({ request, mesg, jc }); + } +} + +async function handleApiRequest({ request, mesg, jc }) { + let resp; + try { + const { name, args } = request as any; + console.log("handling browser.api request:", { name }); + resp = (await getResponse({ name, args })) ?? null; + } catch (err) { + resp = { error: `${err}` }; + } + mesg.respond(jc.encode(resp)); +} + +import * as system from "./system"; + +export const browserApi: BrowserApi = { + system, +}; + +async function getResponse({ name, args }) { + const [group, functionName] = name.split("."); + const f = browserApi[group]?.[functionName]; + if (f == null) { + throw Error(`unknown function '${name}'`); + } + return await f(...args); +} diff --git a/src/packages/frontend/nats/api/system.ts b/src/packages/frontend/nats/api/system.ts new file mode 100644 index 0000000000..ef63739f0d --- /dev/null +++ b/src/packages/frontend/nats/api/system.ts @@ -0,0 +1,10 @@ +export async function ping() { + return { now: Date.now() }; +} + +import { version as versionNumber } from "@cocalc/util/smc-version"; +export async function version() { + return versionNumber; +} + + diff --git a/src/packages/frontend/client/nats.ts b/src/packages/frontend/nats/client.ts similarity index 86% rename from src/packages/frontend/client/nats.ts rename to src/packages/frontend/nats/client.ts index fb2da1bef3..6f8348022a 100644 --- a/src/packages/frontend/client/nats.ts +++ b/src/packages/frontend/nats/client.ts @@ -1,17 +1,18 @@ import * as nats from "nats.ws"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; -import type { WebappClient } from "./client"; +import type { WebappClient } from "@cocalc/frontend/client/client"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { join } from "path"; import * as jetstream from "@nats-io/jetstream"; import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; import { randomId } from "@cocalc/nats/names"; -import { projectSubject } from "@cocalc/nats/names"; +import { browserSubject, projectSubject } from "@cocalc/nats/names"; import { parse_query } from "@cocalc/sync/table/util"; import { sha1 } from "@cocalc/util/misc"; import { keys } from "lodash"; import { type HubApi, initHubApi } from "@cocalc/nats/hub-api"; import { type ProjectApi, initProjectApi } from "@cocalc/nats/project-api"; +import { type BrowserApi, initBrowserApi } from "@cocalc/nats/browser-api"; import { getPrimusConnection } from "@cocalc/nats/primus"; import { isValidUUID } from "@cocalc/util/misc"; import { OpenFiles } from "@cocalc/nats/sync/open-files"; @@ -19,9 +20,11 @@ import { PubSub } from "@cocalc/nats/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; import { SystemKv } from "@cocalc/nats/system"; import { KV } from "@cocalc/nats/sync/kv"; +import { initApi } from "@cocalc/frontend/nats/api"; +import { delay } from "awaiting"; export class NatsClient { - /*private*/ client: WebappClient; + client: WebappClient; private sc = nats.StringCodec(); private jc = nats.JSONCodec(); private nc?: Awaited>; @@ -35,8 +38,18 @@ export class NatsClient { constructor(client: WebappClient) { this.client = client; this.hub = initHubApi(this.callHub); + this.initBrowserApi(); } + private initBrowserApi = async () => { + await delay(100); + try { + await initApi(); + } catch (err) { + console.warn("ERROR -- failed to initialize browser api", err); + } + }; + getConnection = reuseInFlight(async () => { if (this.nc != null) { // undocumented API @@ -163,6 +176,53 @@ export class NatsClient { return this.jc.decode(resp.data); }; + private callBrowser = async ({ + service = "api", + sessionId, + name, + args = [], + timeout = 5000, + }: { + service?: string; + sessionId: string; + name: string; + args: any[]; + timeout?: number; + }) => { + const nc = await this.getConnection(); + const subject = browserSubject({ + account_id: this.client.account_id, + sessionId, + service, + }); + const mesg = this.jc.encode({ + name, + args, + }); + console.log("request to subject", { subject, name, args }); + const resp = await nc.request(subject, mesg, { timeout }); + return this.jc.decode(resp.data); + }; + + browserApi = ({ + sessionId, + timeout, + }: { + sessionId: string; + timeout?: number; + }): BrowserApi => { + const callBrowserApi = async ({ name, args }) => { + return await this.callBrowser({ + sessionId, + timeout, + service: "api", + name, + args, + }); + }; + return initBrowserApi(callBrowserApi); + }; + request = async (subject: string, data: string) => { const c = await this.getConnection(); const resp = await c.request(subject, this.sc.encode(data)); diff --git a/src/packages/nats/browser-api/index.ts b/src/packages/nats/browser-api/index.ts new file mode 100644 index 0000000000..16c9326856 --- /dev/null +++ b/src/packages/nats/browser-api/index.ts @@ -0,0 +1,48 @@ +/* +Request/response API that runs in each browser client. + +DEVELOPMENT: + +Refresh your browser and do this in the console to connect to your own browser: + + > a = cc.client.nats_client.browserApi({sessionId:cc.client.nats_client.sessionId}) + +Then try everything. + +You can also open a second browser tab (with the same account), view the sessionId + + > cc.client.nats_client.sessionId + +then connect from one to the other using that sessionId. This way you can coordinate +between different browsers. +*/ + +import { type System, system } from "./system"; +import { handleErrorMessage } from "@cocalc/nats/util"; + +export interface BrowserApi { + system: System; +} + +const BrowserApiStructure = { + system, +} as const; + +export function initBrowserApi(callBrowserApi): BrowserApi { + const browserApi: any = {}; + for (const group in BrowserApiStructure) { + if (browserApi[group] == null) { + browserApi[group] = {}; + } + for (const functionName in BrowserApiStructure[group]) { + browserApi[group][functionName] = async (...args) => + handleErrorMessage( + await callBrowserApi({ + name: `${group}.${functionName}`, + args, + }), + ); + } + } + return browserApi as BrowserApi; +} diff --git a/src/packages/nats/browser-api/system.ts b/src/packages/nats/browser-api/system.ts new file mode 100644 index 0000000000..155a87e344 --- /dev/null +++ b/src/packages/nats/browser-api/system.ts @@ -0,0 +1,9 @@ +export const system = { + version: true, + ping: true, +}; + +export interface System { + version: () => Promise; + ping: () => Promise<{ now: number }>; +} diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts index d9372e1a4f..7d6cde9564 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/nats/names.ts @@ -20,8 +20,6 @@ export function randomId() { return generateVouchers({ count: 1, length: 10 })[0]; } - - export function projectSubject({ project_id, compute_server_id = 0, @@ -67,3 +65,7 @@ export function projectStreamName({ } return streamName; } + +export function browserSubject({ account_id, sessionId, service }) { + return `${sessionId}.account-${account_id}.${service}`; +} diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 6eafc92ba0..94108bdd88 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -7,6 +7,7 @@ "./hub-api": "./dist/hub-api/index.js", "./hub-api/*": "./dist/hub-api/*.js", "./project-api": "./dist/project-api/index.js", + "./browser-api": "./dist/browser-api/index.js", "./*": "./dist/*.js" }, "scripts": { From c46e0c3517c4e6d26b186daeac7150c825831591 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 5 Feb 2025 16:02:41 +0000 Subject: [PATCH 123/281] nats: clean up how sign in cookies are set; also set a new account_id cookie --- src/packages/backend/auth/cookie-names.ts | 20 ++++++-- src/packages/database/nats/changefeeds.ts | 4 +- src/packages/frontend/client/client.ts | 15 ++++-- src/packages/frontend/nats/api/index.ts | 7 +-- src/packages/frontend/nats/client.ts | 5 +- .../hub/proxy/strip-remember-me-cookie.ts | 14 ++++- src/packages/hub/servers/nats.ts | 51 ++++++------------- src/packages/nats/names.ts | 15 ++++++ .../next/pages/api/v2/accounts/sign-out.ts | 2 + .../next/pages/api/v2/auth/sign-in.ts | 23 +++------ src/packages/server/auth/impersonate.ts | 15 ++---- src/packages/server/auth/set-nats-cookie.ts | 24 +++++++++ .../server/auth/set-sign-in-cookies.ts | 50 ++++++++++++++++++ .../server/auth/sso/passport-login.ts | 23 +++------ src/packages/server/nats/api/index.ts | 6 +-- src/packages/server/nats/index.ts | 4 +- src/packages/sync/table/changefeed-nats.ts | 5 +- src/packages/util/db-schema/accounts.ts | 2 + src/packages/util/misc.ts | 22 +++++--- 19 files changed, 193 insertions(+), 114 deletions(-) create mode 100644 src/packages/server/auth/set-nats-cookie.ts create mode 100644 src/packages/server/auth/set-sign-in-cookies.ts diff --git a/src/packages/backend/auth/cookie-names.ts b/src/packages/backend/auth/cookie-names.ts index 7468aa6f29..a192989504 100644 --- a/src/packages/backend/auth/cookie-names.ts +++ b/src/packages/backend/auth/cookie-names.ts @@ -13,6 +13,7 @@ setting the following environment variable: import basePath from "@cocalc/backend/base-path"; import getLogger from "@cocalc/backend/logger"; +import { basePathCookieName } from "@cocalc/util/misc"; const log = getLogger("cookie-names"); @@ -20,17 +21,26 @@ const log = getLogger("cookie-names"); // when the user is signed in. export const REMEMBER_ME_COOKIE_NAME = process.env.COCALC_REMEMBER_ME_COOKIE_NAME ?? - `${basePath.length <= 1 ? "" : encodeURIComponent(basePath)}remember_me`; + basePathCookieName({ basePath, name: "remember_me" }); log.debug("REMEMBER_ME_COOKIE_NAME", REMEMBER_ME_COOKIE_NAME); // Name of user provided api key cookie, with appropriate base path. // This is set by the user when using the api from node.js, especially // via a websocket. -export const API_COOKIE_NAME = - process.env.COCALC_API_COOKIE_NAME ?? - `${basePath.length <= 1 ? "" : encodeURIComponent(basePath)}api_key`; +export const API_COOKIE_NAME = basePathCookieName({ + basePath, + name: "api_key", +}); log.debug("API_COOKIE_NAME", API_COOKIE_NAME); -export const NATS_JWT_COOKIE_NAME = `${basePath.length <= 1 ? "" : encodeURIComponent(basePath)}cocalc_nats_jwt_cookie`; +export const NATS_JWT_COOKIE_NAME = basePathCookieName({ + basePath, + name: "nats_jwt_cookie", +}); + +export const ACCOUNT_ID_COOKIE_NAME = basePathCookieName({ + basePath, + name: "account_id", +}); diff --git a/src/packages/database/nats/changefeeds.ts b/src/packages/database/nats/changefeeds.ts index c269536061..eadc2dfcee 100644 --- a/src/packages/database/nats/changefeeds.ts +++ b/src/packages/database/nats/changefeeds.ts @@ -2,11 +2,11 @@ 1. turn off nats-server handling for the hub by sending this message from a browser as an admin: - await cc.client.nats_client.hub.system.terminate({service:'database'}) + await cc.client.nats_client.hub.system.terminate({service:'changefeeds'}) 2. Run this - echo "require('@cocalc/database/nats').init()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node + echo "require('@cocalc/database/nats/changefeeds').init()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node */ diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index cabfff8404..49eb0b66a6 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -28,6 +28,10 @@ import { version } from "@cocalc/util/smc-version"; import { setup_global_cocalc } from "./console"; import { Query } from "@cocalc/sync/table"; import debug from "debug"; +import Cookies from "js-cookie"; +import { basePathCookieName } from "@cocalc/util/misc"; +import { ACCOUNT_ID_COOKIE_NAME } from "@cocalc/util/db-schema/accounts"; +import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; // This DEBUG variable comes from webpack: declare const DEBUG; @@ -41,11 +45,17 @@ const log = debug("cocalc"); // all the sync activity logging and everything that calls // client.dbg. +const ACCOUNT_ID_COOKIE = decodeURIComponent( + basePathCookieName({ + basePath: appBasePath, + name: ACCOUNT_ID_COOKIE_NAME, + }), +); + export type AsyncCall = (opts: object) => Promise; export interface WebappClient extends EventEmitter { account_id?: string; - stripe: StripeClient; project_collaborators: ProjectCollaborators; messages: Messages; @@ -126,7 +136,7 @@ Connection events: */ class Client extends EventEmitter implements WebappClient { - account_id?: string; + account_id: string = Cookies.get(ACCOUNT_ID_COOKIE); stripe: StripeClient; project_collaborators: ProjectCollaborators; messages: Messages; @@ -188,7 +198,6 @@ class Client extends EventEmitter implements WebappClient { constructor() { super(); - if (DEBUG) { this.dbg = this.dbg.bind(this); } else { diff --git a/src/packages/frontend/nats/api/index.ts b/src/packages/frontend/nats/api/index.ts index aef7172de6..41c6b9f35a 100644 --- a/src/packages/frontend/nats/api/index.ts +++ b/src/packages/frontend/nats/api/index.ts @@ -5,15 +5,12 @@ import { webapp_client } from "@cocalc/frontend/webapp-client"; import { type BrowserApi } from "@cocalc/nats/browser-api"; import { browserSubject } from "@cocalc/nats/names"; -import { delay } from "awaiting"; export async function initApi() { console.log("init nats browser api - x"); const sessionId = webapp_client.nats_client.sessionId; - console.log({ sessionId }); - // TODO: neeed to wait for signed in event! - while (!webapp_client.account_id) { - await delay(200); + if (!webapp_client.account_id) { + throw Error("must be signed in"); } const subject = browserSubject({ account_id: webapp_client.account_id, diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 6f8348022a..c2f28d6031 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -21,7 +21,6 @@ import type { ChatOptions } from "@cocalc/util/types/llm"; import { SystemKv } from "@cocalc/nats/system"; import { KV } from "@cocalc/nats/sync/kv"; import { initApi } from "@cocalc/frontend/nats/api"; -import { delay } from "awaiting"; export class NatsClient { client: WebappClient; @@ -38,11 +37,10 @@ export class NatsClient { constructor(client: WebappClient) { this.client = client; this.hub = initHubApi(this.callHub); - this.initBrowserApi(); + setTimeout(this.initBrowserApi, 1); } private initBrowserApi = async () => { - await delay(100); try { await initApi(); } catch (err) { @@ -300,6 +298,7 @@ export class NatsClient { args: [{ changes: true, query }], }); } catch (err) { + console.log("changefeedInterest -- error", query); if (noError) { console.warn(err); return; diff --git a/src/packages/hub/proxy/strip-remember-me-cookie.ts b/src/packages/hub/proxy/strip-remember-me-cookie.ts index 1ba747cc8b..cfbbeb98c7 100644 --- a/src/packages/hub/proxy/strip-remember-me-cookie.ts +++ b/src/packages/hub/proxy/strip-remember-me-cookie.ts @@ -9,6 +9,7 @@ auth credentials for all users of a project! import { REMEMBER_ME_COOKIE_NAME, + NATS_JWT_COOKIE_NAME, API_COOKIE_NAME, } from "@cocalc/backend/auth/cookie-names"; @@ -16,13 +17,20 @@ export default function stripRememberMeCookie(cookie): { cookie: string; remember_me: string | undefined; // the value of the cookie we just stripped out. api_key: string | undefined; + nats_jwt: string | undefined; } { if (cookie == null) { - return { cookie, remember_me: undefined, api_key: undefined }; + return { + cookie, + remember_me: undefined, + api_key: undefined, + nats_jwt: undefined, + }; } else { const v: string[] = []; let remember_me: string | undefined = undefined; let api_key: string | undefined = undefined; + let nats_jwt: string | undefined = undefined; for (const c of cookie.split(";")) { const z = c.split("="); if (z[0].trim() == REMEMBER_ME_COOKIE_NAME) { @@ -32,10 +40,12 @@ export default function stripRememberMeCookie(cookie): { remember_me = z[1].trim(); } else if (z[0].trim() == API_COOKIE_NAME) { api_key = z[1].trim(); + } else if (z[0].trim() == NATS_JWT_COOKIE_NAME) { + nats_jwt = z[1].trim(); } else { v.push(c); } } - return { cookie: v.join(";"), remember_me, api_key }; + return { cookie: v.join(";"), remember_me, api_key, nats_jwt }; } } diff --git a/src/packages/hub/servers/nats.ts b/src/packages/hub/servers/nats.ts index c60dab7d93..c39745f039 100644 --- a/src/packages/hub/servers/nats.ts +++ b/src/packages/hub/servers/nats.ts @@ -1,28 +1,20 @@ /* -Proof of concept NATS proxy. +NATS WebSocket proxy -- this primarily just directly proxied the nats +websocket server, so outside browsers can connect to it. -We assume there is a NATS server running on localhost with this configuration: +We assume there is a NATS server running on localhost. This gets configured in +dev mode automatically and started via -# server.conf -websocket { - listen: "localhost:8443" - no_tls: true - jwt_cookie: "cocalc_nats_jwt_cookie" -} - -You could start this with - - nats-server -config server.conf +$ cd ~/cocalc/src +$ pnpm nats-server */ import { createProxyServer } from "http-proxy"; import getLogger from "@cocalc/backend/logger"; -import { NATS_JWT_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; -import Cookies from "cookies"; -import { configureNatsUser, getJwt } from "@cocalc/server/nats/auth"; import { type Router } from "express"; import getAccount from "@cocalc/server/auth/get-account"; +import setNatsCookie from "@cocalc/server/auth/set-nats-cookie"; const logger = getLogger("hub:nats"); @@ -37,6 +29,9 @@ export async function proxyNatsWebsocket(req, socket, head) { target, timeout: 3000, }); + // TODO: we could do "const account_id = await getAccount(req);" and thus verify user is signed in + // before even allowing an attempt to connect. However, connecting without a valid JWT cookie + // just results in immediately failure anyways, so there is no need. proxy.ws(req, socket, head); } @@ -44,28 +39,14 @@ export function initNatsServer(router: Router) { router.get("/nats", async (req, res) => { const account_id = await getAccount(req); if (account_id) { - await setNatsCookie(req, res, account_id); + try { + await setNatsCookie({ req, res, account_id }); + res.json({ account_id }); + } catch (err) { + res.json({ error: `${err}` }); + } } else { res.json({ error: "not signed in" }); } }); } - -async function setNatsCookie(req, res, account_id: string) { - try { - const jwt = await getJwt({ account_id }); - // todo: how frequent? - await configureNatsUser({ account_id }); - const cookies = new Cookies(req, res, { secure: true }); - // 6 months -- long is fine now since we support "sign out everywhere" ? - const maxAge = 1000 * 24 * 3600 * 30 * 6; - cookies.set(NATS_JWT_COOKIE_NAME, jwt, { - maxAge, - sameSite: true, - }); - } catch (err) { - res.json({ error: `Problem setting cookie -- ${err.message}.` }); - return; - } - res.json({ account_id }); -} diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts index 7d6cde9564..7b2620afa6 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/nats/names.ts @@ -33,6 +33,9 @@ export function projectSubject({ service?: string; path?: string; }): string { + if (!project_id) { + throw Error("project_id must be set"); + } let subject = `project.${project_id}.${compute_server_id}`; if (service) { subject += "." + service; @@ -56,6 +59,9 @@ export function projectStreamName({ service?: string; path?: string; }): string { + if (!project_id) { + throw Error("project_id must be set"); + } let streamName = `project-${project_id}-${compute_server_id}`; if (service) { streamName += "-" + service; @@ -67,5 +73,14 @@ export function projectStreamName({ } export function browserSubject({ account_id, sessionId, service }) { + if (!sessionId) { + throw Error("sessionId must be set"); + } + if (!account_id) { + throw Error("account_id must be set"); + } + if (!service) { + throw Error("service must be set"); + } return `${sessionId}.account-${account_id}.${service}`; } diff --git a/src/packages/next/pages/api/v2/accounts/sign-out.ts b/src/packages/next/pages/api/v2/accounts/sign-out.ts index 71baff5940..c3237f575c 100644 --- a/src/packages/next/pages/api/v2/accounts/sign-out.ts +++ b/src/packages/next/pages/api/v2/accounts/sign-out.ts @@ -19,6 +19,7 @@ import { AccountSignOutOutputSchema, } from "lib/api/schema/accounts/sign-out"; import { + ACCOUNT_ID_COOKIE_NAME, NATS_JWT_COOKIE_NAME, REMEMBER_ME_COOKIE_NAME, } from "@cocalc/backend/auth/cookie-names"; @@ -47,6 +48,7 @@ async function signOut(req, res): Promise { // also delete any security relevant cookies for safety and to avoid confusion. res.clearCookie(NATS_JWT_COOKIE_NAME); res.clearCookie(REMEMBER_ME_COOKIE_NAME); + res.clearCookie(ACCOUNT_ID_COOKIE_NAME); } export default apiRoute({ diff --git a/src/packages/next/pages/api/v2/auth/sign-in.ts b/src/packages/next/pages/api/v2/auth/sign-in.ts index 7162f73ed3..ad63587eca 100644 --- a/src/packages/next/pages/api/v2/auth/sign-in.ts +++ b/src/packages/next/pages/api/v2/auth/sign-in.ts @@ -19,6 +19,7 @@ Sign in works as follows: import getPool from "@cocalc/database/pool"; import { createRememberMeCookie } from "@cocalc/server/auth/remember-me"; import { + ACCOUNT_ID_COOKIE_NAME, NATS_JWT_COOKIE_NAME, REMEMBER_ME_COOKIE_NAME, } from "@cocalc/backend/auth/cookie-names"; @@ -27,8 +28,8 @@ import Cookies from "cookies"; import getParams from "lib/api/get-params"; import { verify } from "password-hash"; import { Request, Response } from "express"; -// import reCaptcha from "@cocalc/server/auth/recaptcha"; import { getServerSettings } from "@cocalc/database/settings/server-settings"; +import setSignInCookies from "@cocalc/server/auth/set-sign-in-cookies"; export default async function signIn(req: Request, res: Response) { let { email, password } = getParams(req); @@ -80,24 +81,14 @@ export async function getAccount( } export async function signUserIn(req, res, account_id: string): Promise { - let value, ttl_s; try { - ({ value, ttl_s } = await createRememberMeCookie(account_id)); - } catch (err) { - res.json({ error: `Problem creating session cookie -- ${err.message}.` }); - return; - } - try { - const { samesite_remember_me } = await getServerSettings(); - const cookies = new Cookies(req, res, { secure: true }); - cookies.set(REMEMBER_ME_COOKIE_NAME, value, { - maxAge: ttl_s * 1000, - sameSite: samesite_remember_me, + await setSignInCookies({ + req, + res, + account_id, }); - // ensure there is no stale JWT cookie - res.clearCookie(NATS_JWT_COOKIE_NAME); } catch (err) { - res.json({ error: `Problem setting cookie -- ${err.message}.` }); + res.json({ error: `Problem setting auth cookies -- ${err}` }); return; } res.json({ account_id }); diff --git a/src/packages/server/auth/impersonate.ts b/src/packages/server/auth/impersonate.ts index 96cd3ecabe..72e82f5c3c 100644 --- a/src/packages/server/auth/impersonate.ts +++ b/src/packages/server/auth/impersonate.ts @@ -1,15 +1,12 @@ /* Sign in using an impersonation auth_token. */ -import Cookies from "cookies"; - -import { REMEMBER_ME_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; import base_path from "@cocalc/backend/base-path"; import getPool from "@cocalc/database/pool"; import { getServerSettings } from "@cocalc/database/settings/server-settings"; import clientSideRedirect from "@cocalc/server/auth/client-side-redirect"; -import { createRememberMeCookie } from "@cocalc/server/auth/remember-me"; import { isLocale } from "@cocalc/util/i18n/const"; import { join } from "path"; +import setSignInCookies from "@cocalc/server/auth/set-sign-in-cookies"; export async function signInUsingImpersonateToken({ req, res }) { try { @@ -33,14 +30,8 @@ async function doIt({ req, res }) { throw Error(`unknown or expired token: '${auth_token}'`); } const { account_id } = rows[0]; - - const { value, ttl_s } = await createRememberMeCookie(account_id, 12 * 3600); - const cookies = new Cookies(req, res); - const { samesite_remember_me } = await getServerSettings(); - cookies.set(REMEMBER_ME_COOKIE_NAME, value, { - maxAge: ttl_s * 1000, - sameSite: samesite_remember_me, - }); + // maxAge = 12 hours + await setSignInCookies({ req, res, account_id, maxAge: 12 * 3600 * 1000 }); const { dns } = await getServerSettings(); let target = `https://${dns}${join(base_path, "app")}`; diff --git a/src/packages/server/auth/set-nats-cookie.ts b/src/packages/server/auth/set-nats-cookie.ts new file mode 100644 index 0000000000..af6d22175a --- /dev/null +++ b/src/packages/server/auth/set-nats-cookie.ts @@ -0,0 +1,24 @@ +import { NATS_JWT_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; +import Cookies from "cookies"; +import { configureNatsUser, getJwt } from "@cocalc/server/nats/auth"; +import { DEFAULT_MAX_AGE_MS } from "./set-sign-in-cookies"; + +export default async function setNatsCookie({ + req, + res, + account_id, + maxAge = DEFAULT_MAX_AGE_MS, +}: { + req; + res; + account_id: string; + maxAge?: number; +}) { + const jwt = await getJwt({ account_id }); + await configureNatsUser({ account_id }); + const cookies = new Cookies(req, res, { secure: true }); + cookies.set(NATS_JWT_COOKIE_NAME, jwt, { + maxAge, + sameSite: true, + }); +} diff --git a/src/packages/server/auth/set-sign-in-cookies.ts b/src/packages/server/auth/set-sign-in-cookies.ts new file mode 100644 index 0000000000..1ac8f265c2 --- /dev/null +++ b/src/packages/server/auth/set-sign-in-cookies.ts @@ -0,0 +1,50 @@ +import { + ACCOUNT_ID_COOKIE_NAME, + REMEMBER_ME_COOKIE_NAME, +} from "@cocalc/backend/auth/cookie-names"; +import { createRememberMeCookie } from "@cocalc/server/auth/remember-me"; +import setNatsCookie from "@cocalc/server/auth/set-nats-cookie"; +import { getServerSettings } from "@cocalc/database/settings/server-settings"; +import Cookies from "cookies"; + +// 6 months by default, but sometimes (e.g., impersonate) is MUCH shorter. +export const DEFAULT_MAX_AGE_MS = 24 * 3600 * 30 * 1000 * 6; + +export default async function setSignInCookies({ + req, + res, + account_id, + maxAge = DEFAULT_MAX_AGE_MS, +}: { + req; + res; + account_id: string; + maxAge?: number; +}) { + const opts = { req, res, account_id, maxAge }; + await Promise.all([ + setRememberMeCookie(opts), + setNatsCookie(opts), + setAccountIdCookie(opts), + ]); +} + +async function setRememberMeCookie({ req, res, account_id, maxAge }) { + const { value } = await createRememberMeCookie(account_id, maxAge / 1000); + const cookies = new Cookies(req, res); + const { samesite_remember_me } = await getServerSettings(); + cookies.set(REMEMBER_ME_COOKIE_NAME, value, { + maxAge, + sameSite: samesite_remember_me, + }); +} + +async function setAccountIdCookie({ req, res, account_id, maxAge }) { + // account_id cookie is NOT secure since user is supposed to read it + // from browser. It's not for telling the server the account_id, but + // for telling the user their own account_id. + const cookies = new Cookies(req, res, { secure: false, httpOnly: false }); + cookies.set(ACCOUNT_ID_COOKIE_NAME, account_id, { + maxAge, + }); +} diff --git a/src/packages/server/auth/sso/passport-login.ts b/src/packages/server/auth/sso/passport-login.ts index 3a5115a1ac..f923b999d0 100644 --- a/src/packages/server/auth/sso/passport-login.ts +++ b/src/packages/server/auth/sso/passport-login.ts @@ -27,7 +27,6 @@ import { set_email_address_verified } from "@cocalc/database/postgres/account-qu import type { PostgreSQL } from "@cocalc/database/postgres/types"; import generateHash from "@cocalc/server/auth/hash"; import { REMEMBER_ME_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; -import { createRememberMeCookie } from "@cocalc/server/auth/remember-me"; import { sanitizeID } from "@cocalc/server/auth/sso/sanitize-id"; import { sanitizeProfile } from "@cocalc/server/auth/sso/sanitize-profile"; import { @@ -43,7 +42,7 @@ import { SSO_API_KEY_COOKIE_NAME } from "./consts"; import isBanned from "@cocalc/server/accounts/is-banned"; import accountCreationActions from "@cocalc/server/accounts/account-creation-actions"; import clientSideRedirect from "@cocalc/server/auth/client-side-redirect"; -import { getServerSettings } from "@cocalc/database/settings/server-settings"; +import setSignInCookies from "@cocalc/server/auth/set-sign-in-cookies"; const logger = getLogger("server:auth:sso:passport-login"); @@ -456,7 +455,6 @@ export class PassportLogin { }); } - // optionally, SSO strategies can be configured to always update fields of the user // with the data they provide. right now that's first and last name. // email address is a bit more tricky and not implemented. @@ -488,7 +486,6 @@ export class PassportLogin { }); } - // ebfore recording the sign-in below, we check if a user is banned private async isUserBanned(account_id, email_address): Promise { const is_banned = await isBanned(account_id); @@ -506,7 +503,7 @@ export class PassportLogin { // SSO strategies can configure the expiration of that cookie – e.g. super // paranoid ones can set this to 1 day. private async handleNewSignIn( - opts: PassportLoginOpts, + { req, res }: PassportLoginOpts, locals: PassportLoginLocals, ): Promise { if (locals.has_valid_remember_me) return; @@ -519,17 +516,11 @@ export class PassportLogin { L("passport created: set remember_me cookie, so user gets logged in"); - L(`create remember_me cookie in database: ttl=${opts.cookie_ttl_s}s`); - const { value, ttl_s } = await createRememberMeCookie( - locals.account_id, - opts.cookie_ttl_s, - ); - - L(`actually set remember_me cookie in client. ttl=${ttl_s}s`); - const { samesite_remember_me } = await getServerSettings(); - locals.cookies.set(REMEMBER_ME_COOKIE_NAME, value, { - maxAge: ttl_s * 1000, - sameSite: samesite_remember_me, + L(`create remember_me cookie in database`); + await setSignInCookies({ + account_id: locals.account_id, + req, + res, }); } } diff --git a/src/packages/server/nats/api/index.ts b/src/packages/server/nats/api/index.ts index 50de6a6356..3b97f106b0 100644 --- a/src/packages/server/nats/api/index.ts +++ b/src/packages/server/nats/api/index.ts @@ -40,7 +40,7 @@ import getLogger from "@cocalc/backend/logger"; import { type HubApi, getUserId, transformArgs } from "@cocalc/nats/hub-api"; import { getConnection } from "@cocalc/backend/nats"; import userIsInGroup from "@cocalc/server/accounts/is-in-group"; -import { terminate as terminateDatabase } from "@cocalc/database/nats/changefeeds"; +import { terminate as terminateChangefeeds } from "@cocalc/database/nats/changefeeds"; const logger = getLogger("server:nats:api"); @@ -66,8 +66,8 @@ export async function initAPI() { // one case halts this loop const { service } = request.args[0] ?? {}; logger.debug(`Terminate service '${service}'`); - if (service == "database") { - terminateDatabase(); + if (service == "changefeeds") { + terminateChangefeeds(); mesg.respond(jc.encode({ status: "terminated", service })); continue; } else if (service == "api") { diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts index cffb42b33e..b458a1c226 100644 --- a/src/packages/server/nats/index.ts +++ b/src/packages/server/nats/index.ts @@ -1,6 +1,6 @@ import getLogger from "@cocalc/backend/logger"; import { initAPI } from "./api"; -import { init as initDatabase } from "@cocalc/database/nats/changefeeds"; +import { init as initChangefeeds } from "@cocalc/database/nats/changefeeds"; const logger = getLogger("server:nats"); @@ -8,5 +8,5 @@ export default async function initNatsServer() { logger.debug("initializing nats cocalc hub server"); // do NOT await this! initAPI(); - initDatabase(); + initChangefeeds(); } diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index 891ce3b108..33befd499e 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -33,10 +33,7 @@ export class NatsChangefeed extends EventEmitter { }; close = (): void => { - if (this.watch != null) { - this.watch.stop(); - delete this.watch; - } + this.natsSynctable.close(); this.state = "closed"; this.emit("close"); }; diff --git a/src/packages/util/db-schema/accounts.ts b/src/packages/util/db-schema/accounts.ts index 58b7ddf815..8324618c4c 100644 --- a/src/packages/util/db-schema/accounts.ts +++ b/src/packages/util/db-schema/accounts.ts @@ -917,3 +917,5 @@ export interface UserSearchResult { // of users queried by substring searches, obviously. email_address?: string; } + +export const ACCOUNT_ID_COOKIE_NAME = 'account_id'; diff --git a/src/packages/util/misc.ts b/src/packages/util/misc.ts index 98318c7f1d..454a6d8920 100644 --- a/src/packages/util/misc.ts +++ b/src/packages/util/misc.ts @@ -70,12 +70,12 @@ import sha1 from "sha1"; export { sha1 }; function base16ToBase64(hex) { - return Buffer.from(hex, 'hex').toString('base64') -// let bytes: number[] = []; -// for (let c = 0; c < hex.length; c += 2) { -// bytes.push(parseInt(hex.substr(c, 2), 16)); -// } -// return btoa(String.fromCharCode.apply(null, bytes)); + return Buffer.from(hex, "hex").toString("base64"); + // let bytes: number[] = []; + // for (let c = 0; c < hex.length; c += 2) { + // bytes.push(parseInt(hex.substr(c, 2), 16)); + // } + // return btoa(String.fromCharCode.apply(null, bytes)); } export function sha1base64(s) { @@ -2685,3 +2685,13 @@ export function tail(s: string, lines: number) { // Return the substring starting from the next character after the last newline return s.slice(lastIndex + 2); } + +export function basePathCookieName({ + basePath, + name, +}: { + basePath: string; + name: string; +}): string { + return `${basePath.length <= 1 ? "" : encodeURIComponent(basePath)}${name}`; +} From 2ef3d162f903f9d348c82bba298ab2e1ece9c032 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 5 Feb 2025 17:42:28 +0000 Subject: [PATCH 124/281] nats: fix changefeeds (they were hanging due to a refactor) --- src/packages/database/nats/changefeeds.ts | 21 ++++++++------- src/packages/frontend/nats/client.ts | 32 ++++++++++++++--------- src/packages/server/nats/api/index.ts | 6 ++--- src/packages/server/nats/index.ts | 4 +-- 4 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/packages/database/nats/changefeeds.ts b/src/packages/database/nats/changefeeds.ts index eadc2dfcee..4b9707b731 100644 --- a/src/packages/database/nats/changefeeds.ts +++ b/src/packages/database/nats/changefeeds.ts @@ -2,11 +2,10 @@ 1. turn off nats-server handling for the hub by sending this message from a browser as an admin: - await cc.client.nats_client.hub.system.terminate({service:'changefeeds'}) + await cc.client.nats_client.hub.system.terminate({service:'db'}) 2. Run this - echo "require('@cocalc/database/nats/changefeeds').init()" | COCALC_MODE='single-user' DEBUG_CONSOLE=yes DEBUG=cocalc:* node */ @@ -29,7 +28,7 @@ import { delay } from "awaiting"; import type { Subscription } from "nats"; import { debounce } from "lodash"; -const logger = getLogger("database:nats"); +const logger = getLogger("database:nats:changefeeds"); const DEBOUNCE_SAVE_TO_JETSTREAM = 100; const MAX_TIME_SAVE_TO_JETSTREAM = 30000; @@ -61,19 +60,21 @@ async function handleRequest(mesg, nc) { try { const { account_id, project_id } = getUserId(mesg.subject); const { name, args } = jc.decode(mesg.data) ?? ({} as any); + // logger.debug(`got request: "${JSON.stringify({ name, args })}"`); if (!name) { throw Error("api endpoint name must be given in message"); } - logger.debug("handling database request:", { - account_id, - project_id, - name, - //args, - }); + // logger.debug("handling server='db' request:", { + // account_id, + // project_id, + // name, + // }); resp = await getResponse({ name, args, account_id, project_id, nc }); } catch (err) { + // logger.debug("ERROR", err); resp = { error: `${err}` }; } + // logger.debug(`Responding with "${JSON.stringify(resp)}"`); mesg.respond(jc.encode(resp)); } @@ -258,6 +259,7 @@ const createChangefeed = reuseInFlight( setMap(synctable.getKey(obj), obj); } processQueue(); + cb(); }; const handleUpdate = ({ action, new_val, old_val }) => { @@ -315,6 +317,7 @@ const createChangefeed = reuseInFlight( } } }; + // do not block on this. watch(); return; diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index c2f28d6031..d7258f96e6 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -21,6 +21,7 @@ import type { ChatOptions } from "@cocalc/util/types/llm"; import { SystemKv } from "@cocalc/nats/system"; import { KV } from "@cocalc/nats/sync/kv"; import { initApi } from "@cocalc/frontend/nats/api"; +import { delay } from "awaiting"; export class NatsClient { client: WebappClient; @@ -37,10 +38,12 @@ export class NatsClient { constructor(client: WebappClient) { this.client = client; this.hub = initHubApi(this.callHub); - setTimeout(this.initBrowserApi, 1); + this.initBrowserApi(); } private initBrowserApi = async () => { + // have to delay so that this.client is fully created. + await delay(1); try { await initApi(); } catch (err) { @@ -101,15 +104,20 @@ export class NatsClient { }) => { const nc = await this.getConnection(); const subject = `hub.account.${this.client.account_id}.${service}`; - const resp = await nc.request( - subject, - this.jc.encode({ - name, - args, - }), - { timeout }, - ); - return this.jc.decode(resp.data); + try { + const resp = await nc.request( + subject, + this.jc.encode({ + name, + args, + }), + { timeout }, + ); + return this.jc.decode(resp.data); + } catch (err) { + err.message = `${err.message} - callHub: subject='${subject}', name='${name}', `; + throw err; + } }; // Returns api for RPC calls to the project with typescript support! @@ -197,7 +205,7 @@ export class NatsClient { name, args, }); - console.log("request to subject", { subject, name, args }); + // console.log("request to subject", { subject, name, args }); const resp = await nc.request(subject, mesg, { timeout }); return this.jc.decode(resp.data); }; @@ -298,8 +306,8 @@ export class NatsClient { args: [{ changes: true, query }], }); } catch (err) { - console.log("changefeedInterest -- error", query); if (noError) { + console.warn("changefeedInterest -- error", query); console.warn(err); return; } else { diff --git a/src/packages/server/nats/api/index.ts b/src/packages/server/nats/api/index.ts index 3b97f106b0..682ff61461 100644 --- a/src/packages/server/nats/api/index.ts +++ b/src/packages/server/nats/api/index.ts @@ -40,7 +40,7 @@ import getLogger from "@cocalc/backend/logger"; import { type HubApi, getUserId, transformArgs } from "@cocalc/nats/hub-api"; import { getConnection } from "@cocalc/backend/nats"; import userIsInGroup from "@cocalc/server/accounts/is-in-group"; -import { terminate as terminateChangefeeds } from "@cocalc/database/nats/changefeeds"; +import { terminate as terminateDatabase } from "@cocalc/database/nats/changefeeds"; const logger = getLogger("server:nats:api"); @@ -66,8 +66,8 @@ export async function initAPI() { // one case halts this loop const { service } = request.args[0] ?? {}; logger.debug(`Terminate service '${service}'`); - if (service == "changefeeds") { - terminateChangefeeds(); + if (service == "db") { + terminateDatabase(); mesg.respond(jc.encode({ status: "terminated", service })); continue; } else if (service == "api") { diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts index b458a1c226..cffb42b33e 100644 --- a/src/packages/server/nats/index.ts +++ b/src/packages/server/nats/index.ts @@ -1,6 +1,6 @@ import getLogger from "@cocalc/backend/logger"; import { initAPI } from "./api"; -import { init as initChangefeeds } from "@cocalc/database/nats/changefeeds"; +import { init as initDatabase } from "@cocalc/database/nats/changefeeds"; const logger = getLogger("server:nats"); @@ -8,5 +8,5 @@ export default async function initNatsServer() { logger.debug("initializing nats cocalc hub server"); // do NOT await this! initAPI(); - initChangefeeds(); + initDatabase(); } From fcf157c40e1d4f239c6d76cf53f8e36c49689921 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 6 Feb 2025 00:38:26 +0000 Subject: [PATCH 125/281] nats: make my req/reply api services be registered NATS microservices, including from the browsers and projects - right now this is WILDLY INSECURE --- src/packages/database/nats/changefeeds.ts | 27 ++++++++++++++---- src/packages/database/package.json | 6 +++- src/packages/frontend/nats/api/index.ts | 28 ++++++++++++------- src/packages/frontend/nats/api/system.ts | 6 ++-- src/packages/frontend/nats/client.ts | 8 ++++++ src/packages/frontend/package.json | 1 + src/packages/nats/names.ts | 1 + src/packages/pnpm-lock.yaml | 27 ++++++++++++++++++ src/packages/project/nats/api/index.ts | 24 +++++++++++----- src/packages/project/package.json | 5 +++- .../server/auth/set-sign-in-cookies.ts | 1 + src/packages/server/nats/api/index.ts | 16 +++++++++-- src/packages/server/nats/auth.ts | 4 +++ src/packages/server/package.json | 1 + 14 files changed, 125 insertions(+), 30 deletions(-) diff --git a/src/packages/database/nats/changefeeds.ts b/src/packages/database/nats/changefeeds.ts index 4b9707b731..102d575961 100644 --- a/src/packages/database/nats/changefeeds.ts +++ b/src/packages/database/nats/changefeeds.ts @@ -25,8 +25,9 @@ import jsonStableStringify from "json-stable-stringify"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { uuid } from "@cocalc/util/misc"; import { delay } from "awaiting"; -import type { Subscription } from "nats"; import { debounce } from "lodash"; +import { Svcm, type ServiceMsg } from "@nats-io/services"; +import { type QueuedIterator } from "nats"; const logger = getLogger("database:nats:changefeeds"); @@ -35,22 +36,33 @@ const MAX_TIME_SAVE_TO_JETSTREAM = 30000; const jc = JSONCodec(); -let subscription: Subscription | null = null; +let api: QueuedIterator | null = null; export async function init() { const subject = "hub.*.*.db"; logger.debug(`init -- subject='${subject}', options=`, { queue: "0", }); const nc = await getConnection(); - subscription = nc.subscribe(subject, { queue: "0" }); - for await (const mesg of subscription) { + + // @ts-ignore + const svcm = new Svcm(nc); + + const service = await svcm.add({ + name: "db-server", + version: "0.1.0", + description: "CoCalc Database Service (changefeeds)", + }); + + const api = service.addEndpoint("api", { subject }); + + for await (const mesg of api) { handleRequest(mesg, nc); } } export function terminate() { logger.debug("terminating"); - subscription?.unsubscribe(); + api?.stop(); // also, stop reporting data into the streams cancelAllChangefeeds(); } @@ -265,6 +277,11 @@ const createChangefeed = reuseInFlight( const handleUpdate = ({ action, new_val, old_val }) => { // action = 'insert', 'update', 'delete', 'close' // e.g., {"action":"insert","new_val":{"title":"testingxxxxx","project_id":"81e0c408-ac65-4114-bad5-5f4b6539bd0e"}} + const obj = new_val ?? old_val; + if (obj == null) { + // nothing we can do with this + return; + } const key = synctable.getKey(new_val ?? old_val); if (action == "insert") { setMap(key, new_val); diff --git a/src/packages/database/package.json b/src/packages/database/package.json index 6e1443c98b..0642d0a0ea 100644 --- a/src/packages/database/package.json +++ b/src/packages/database/package.json @@ -22,6 +22,7 @@ "@cocalc/database": "workspace:*", "@cocalc/nats": "workspace:*", "@cocalc/util": "workspace:*", + "@nats-io/services": "3.0.0-25", "@types/lodash": "^4.14.202", "@types/pg": "^8.6.1", "@types/uuid": "^8.3.1", @@ -59,7 +60,10 @@ "url": "https://github.com/sagemathinc/cocalc" }, "homepage": "https://github.com/sagemathinc/cocalc", - "keywords": ["postgresql", "cocalc"], + "keywords": [ + "postgresql", + "cocalc" + ], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "bugs": { diff --git a/src/packages/frontend/nats/api/index.ts b/src/packages/frontend/nats/api/index.ts index 41c6b9f35a..3a39a2f412 100644 --- a/src/packages/frontend/nats/api/index.ts +++ b/src/packages/frontend/nats/api/index.ts @@ -4,28 +4,36 @@ import { webapp_client } from "@cocalc/frontend/webapp-client"; import { type BrowserApi } from "@cocalc/nats/browser-api"; +import { Svcm } from "@nats-io/services"; import { browserSubject } from "@cocalc/nats/names"; export async function initApi() { console.log("init nats browser api - x"); - const sessionId = webapp_client.nats_client.sessionId; - if (!webapp_client.account_id) { + const { account_id } = webapp_client; + if (!account_id) { throw Error("must be signed in"); } + const { sessionId } = webapp_client.nats_client; + console.log("create browser microservice"); + const { jc, nc } = await webapp_client.nats_client.getEnv(); + // @ts-ignore + const svcm = new Svcm(nc); const subject = browserSubject({ - account_id: webapp_client.account_id, + account_id, sessionId, service: "api", }); - console.log({ subject }); - console.log("init browser API", { sessionId, subject }); - const { jc, nc } = await webapp_client.nats_client.getEnv(); - const subscription = nc.subscribe(subject); - listen({ subscription, jc }); + const service = await svcm.add({ + name: `account-${account_id}`, + version: "0.1.0", + description: "CoCalc Web Browser", + }); + const api = service.addEndpoint("api", { subject }); + listen({ api, jc }); } -async function listen({ subscription, jc }) { - for await (const mesg of subscription) { +async function listen({ api, jc }) { + for await (const mesg of api) { const request = jc.decode(mesg.data); handleApiRequest({ request, mesg, jc }); } diff --git a/src/packages/frontend/nats/api/system.ts b/src/packages/frontend/nats/api/system.ts index ef63739f0d..36e9ccf115 100644 --- a/src/packages/frontend/nats/api/system.ts +++ b/src/packages/frontend/nats/api/system.ts @@ -1,10 +1,10 @@ +import { webapp_client } from "@cocalc/frontend/webapp-client"; + export async function ping() { - return { now: Date.now() }; + return { now: Date.now(), sessionId: webapp_client.nats_client.sessionId }; } import { version as versionNumber } from "@cocalc/util/smc-version"; export async function version() { return versionNumber; } - - diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index d7258f96e6..b965642145 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -22,6 +22,7 @@ import { SystemKv } from "@cocalc/nats/system"; import { KV } from "@cocalc/nats/sync/kv"; import { initApi } from "@cocalc/frontend/nats/api"; import { delay } from "awaiting"; +import { Svcm } from "@nats-io/services"; export class NatsClient { client: WebappClient; @@ -420,4 +421,11 @@ export class NatsClient { return this.kvCache[key]; }, ); + + microservicesClient = async () => { + const nc = await this.getConnection(); + // @ts-ignore + const svcm = new Svcm(nc); + return svcm.client(); + }; } diff --git a/src/packages/frontend/package.json b/src/packages/frontend/package.json index 7422b89439..58d47cf3a3 100644 --- a/src/packages/frontend/package.json +++ b/src/packages/frontend/package.json @@ -60,6 +60,7 @@ "@microlink/react-json-view": "^1.23.3", "@nats-io/jetstream": "3.0.0-36", "@nats-io/kv": "3.0.0-30", + "@nats-io/services": "3.0.0-25", "@orama/orama": "3.0.0-rc-3", "@react-hook/mouse-position": "^4.1.3", "@rinsuki/lz4-ts": "^1.0.1", diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts index 7b2620afa6..c62e9a085a 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/nats/names.ts @@ -72,6 +72,7 @@ export function projectStreamName({ return streamName; } + export function browserSubject({ account_id, sessionId, service }) { if (!sessionId) { throw Error("sessionId must be set"); diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 275ec041a6..d60b9db1fe 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -177,6 +177,9 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util + '@nats-io/services': + specifier: 3.0.0-25 + version: 3.0.0-25 '@types/lodash': specifier: ^4.14.202 version: 4.17.9 @@ -319,6 +322,9 @@ importers: '@nats-io/kv': specifier: 3.0.0-30 version: 3.0.0-30 + '@nats-io/services': + specifier: 3.0.0-25 + version: 3.0.0-25 '@orama/orama': specifier: 3.0.0-rc-3 version: 3.0.0-rc-3 @@ -1287,6 +1293,9 @@ importers: '@nats-io/kv': specifier: 3.0.0-30 version: 3.0.0-30 + '@nats-io/services': + specifier: 3.0.0-25 + version: 3.0.0-25 '@nteract/messaging': specifier: ^7.0.20 version: 7.0.20 @@ -1477,6 +1486,9 @@ importers: '@nats-io/jetstream': specifier: 3.0.0-36 version: 3.0.0-36 + '@nats-io/services': + specifier: 3.0.0-25 + version: 3.0.0-25 '@node-saml/passport-saml': specifier: ^4.0.4 version: 4.0.4 @@ -3939,6 +3951,9 @@ packages: '@nats-io/nats-core@3.0.0-49': resolution: {integrity: sha512-Xe7LjCdhtL4pXk2czwUE8Y1elTy/zo3ZzpoIwOO+/uJPughEsSxCpqygPrDqWcOG2uWVB9G1wxjg8r0Y9StovQ==} + '@nats-io/nats-core@3.0.0-50': + resolution: {integrity: sha512-Kur1/yhzNrpcpu+OhsQ89k9Ge1woEWJd5FV3tpp0BtpRWMlIth3StBiADguynbKSQCkBAOUQ+C0kRnFW8zOIeg==} + '@nats-io/nkeys@2.0.2': resolution: {integrity: sha512-0JTyVl9P+UJyjUBDWP9589TuUKXJQ8tDkVRgi02X/MMzW997+4FykirvZEkIe6ZAhiLIBN+NpN8ULMMt6mDrbA==} engines: {node: '>=18.0.0'} @@ -3947,6 +3962,9 @@ packages: resolution: {integrity: sha512-TpA3HEBna/qMVudy+3HZr5M3mo/L1JPofpVT4t0HkFGkz2Cn9wrlrQC8tvR8Md5Oa9//GtGG26eN0qEWF5Vqew==} engines: {node: '>= 18.x'} + '@nats-io/services@3.0.0-25': + resolution: {integrity: sha512-/wS4kYCT6QzikDUJV/vjlnoXmhrXCyLsjBP0J3PNaxFZeUbpPo2uqqHQTwGRia26vfNvlGHMODSHSNgxw8SkPg==} + '@nestjs/axios@3.0.3': resolution: {integrity: sha512-h6TCn3yJwD6OKqqqfmtRS5Zo4E46Ip2n+gK1sqwzNBC+qxQ9xpCu+ODVRFur6V3alHSCSBxb3nNtt73VEdluyA==} peerDependencies: @@ -14956,12 +14974,21 @@ snapshots: '@nats-io/nkeys': 2.0.2 '@nats-io/nuid': 2.0.3 + '@nats-io/nats-core@3.0.0-50': + dependencies: + '@nats-io/nkeys': 2.0.2 + '@nats-io/nuid': 2.0.3 + '@nats-io/nkeys@2.0.2': dependencies: tweetnacl: 1.0.3 '@nats-io/nuid@2.0.3': {} + '@nats-io/services@3.0.0-25': + dependencies: + '@nats-io/nats-core': 3.0.0-50 + '@nestjs/axios@3.0.3(@nestjs/common@10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1))(axios@1.7.4)(rxjs@7.8.1)': dependencies: '@nestjs/common': 10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1) diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index b4608e9c3b..39de6befac 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -3,7 +3,7 @@ How to do development (so in a dev project doing cc-in-cc dev). 0. From the browser, terminate this api server running in the project already, if any - await cc.client.nats_client.projectApi({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}).system.terminate({service:'api'}) + await cc.client.nats_client.projectApi({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'}).system.terminate({service:'api'}) 1. Open a terminal in the project itself, which sets up the required environment variables, e.g., @@ -39,21 +39,31 @@ import { type ProjectApi } from "@cocalc/nats/project-api"; import getConnection from "@cocalc/project/nats/connection"; import { getSubject } from "../names"; import { terminate as terminateOpenFiles } from "@cocalc/project/nats/open-files"; +import { Svcm } from "@nats-io/services"; +import { compute_server_id, project_id } from "@cocalc/project/data"; const logger = getLogger("project:nats:api"); const jc = JSONCodec(); export async function init() { const subject = getSubject({ service: "api" }); - logger.debug(`initAPI -- subject='${subject}'`); const nc = await getConnection(); - const subscription = nc.subscribe(subject); + // @ts-ignore + const svcm = new Svcm(nc); + const name = `project-${project_id}`; + logger.debug(`creating API microservice ${name}`); + const service = await svcm.add({ + name, + version: "0.1.0", + description: `CoCalc ${compute_server_id ? "Compute Server" : "Project"}`, + }); + const api = service.addEndpoint("api", { subject }); logger.debug(`initAPI -- subscribed to subject='${subject}'`); - listen(subscription, subject); + listen(api, subject); } -async function listen(subscription, subject) { - for await (const mesg of subscription) { +async function listen(api, subject) { + for await (const mesg of api) { const request = jc.decode(mesg.data) ?? ({} as any); // logger.debug("got message", request); if (request.name == "system.terminate") { @@ -69,7 +79,7 @@ async function listen(subscription, subject) { console.warn("TERMINATING listening on ", subject); logger.debug("TERMINATING listening on ", subject); mesg.respond(jc.encode({ status: "terminated", service })); - subscription.unsubscribe(); + api.stop(); return; } else { mesg.respond(jc.encode({ error: `Unknown service ${service}` })); diff --git a/src/packages/project/package.json b/src/packages/project/package.json index 6cb0cc4eeb..f6b7825753 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -34,6 +34,7 @@ "@cocalc/util": "workspace:*", "@nats-io/jetstream": "3.0.0-36", "@nats-io/kv": "3.0.0-30", + "@nats-io/services": "3.0.0-25", "@nteract/messaging": "^7.0.20", "@types/lodash": "^4.14.202", "@types/primus": "^7.3.9", @@ -88,7 +89,9 @@ "clean": "rm -rf dist" }, "author": "SageMath, Inc.", - "contributors": ["William Stein "], + "contributors": [ + "William Stein " + ], "license": "SEE LICENSE.md", "bugs": { "url": "https://github.com/sagemathinc/cocalc/issues" diff --git a/src/packages/server/auth/set-sign-in-cookies.ts b/src/packages/server/auth/set-sign-in-cookies.ts index 1ac8f265c2..6d9b29d1f6 100644 --- a/src/packages/server/auth/set-sign-in-cookies.ts +++ b/src/packages/server/auth/set-sign-in-cookies.ts @@ -46,5 +46,6 @@ async function setAccountIdCookie({ req, res, account_id, maxAge }) { const cookies = new Cookies(req, res, { secure: false, httpOnly: false }); cookies.set(ACCOUNT_ID_COOKIE_NAME, account_id, { maxAge, + httpOnly: false, }); } diff --git a/src/packages/server/nats/api/index.ts b/src/packages/server/nats/api/index.ts index 682ff61461..b0862f5941 100644 --- a/src/packages/server/nats/api/index.ts +++ b/src/packages/server/nats/api/index.ts @@ -41,6 +41,7 @@ import { type HubApi, getUserId, transformArgs } from "@cocalc/nats/hub-api"; import { getConnection } from "@cocalc/backend/nats"; import userIsInGroup from "@cocalc/server/accounts/is-in-group"; import { terminate as terminateDatabase } from "@cocalc/database/nats/changefeeds"; +import { Svcm } from "@nats-io/services"; const logger = getLogger("server:nats:api"); @@ -52,8 +53,17 @@ export async function initAPI() { queue: "0", }); const nc = await getConnection(); - const sub = nc.subscribe(subject, { queue: "0" }); - for await (const mesg of sub) { + // @ts-ignore + const svcm = new Svcm(nc); + + const service = await svcm.add({ + name: "hub-server", + version: "0.1.0", + description: "CoCalc Hub Server", + }); + + const api = service.addEndpoint("api", { subject }); + for await (const mesg of api) { const request = jc.decode(mesg.data) ?? ({} as any); if (request.name == "system.terminate") { // special hook so admin can terminate handling. This is useful for development. @@ -75,7 +85,7 @@ export async function initAPI() { console.warn("TERMINATING listening on ", subject); logger.debug("TERMINATING listening on ", subject); mesg.respond(jc.encode({ status: "terminated", service })); - sub.unsubscribe(); + api.stop(); return; } else { mesg.respond(jc.encode({ error: `Unknown service ${service}` })); diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 7d369944a4..9ab8343a3b 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -114,11 +114,15 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { "_INBOX.>", // so can use request/response `hub.${userType}.${userId}.>`, // can talk as *only this user* to the hub's api's "$JS.API.>", // so can use Jestream: TODO: too much???! + "$SRV.>", // TODO: obviously vastly too general! + ">", ]); const goalSub = new Set([ "_INBOX.>", // so can user request/response "$JS.API.>", // TODO! This needs to be restrained more, I think??! Don't know. "system.>", // access to READ the system info kv store. + "$SRV.>", // TODO: obviously vastly too general! + ">", ]); if (userType == "account") { diff --git a/src/packages/server/package.json b/src/packages/server/package.json index 567fd12afb..a3d91fc595 100644 --- a/src/packages/server/package.json +++ b/src/packages/server/package.json @@ -60,6 +60,7 @@ "@langchain/mistralai": "^0.2.0", "@langchain/openai": "^0.3.17", "@nats-io/jetstream": "3.0.0-36", + "@nats-io/services": "3.0.0-25", "@node-saml/passport-saml": "^4.0.4", "@passport-js/passport-twitter": "^1.0.8", "@passport-next/passport-google-oauth2": "^1.0.0", From c103fd8a97df80e350b2d5303112892dc4fc56d3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 6 Feb 2025 01:49:34 +0000 Subject: [PATCH 126/281] nats: locking down the JWT permissions rules (terminals now broken by this) --- src/packages/nats/sync/synctable-kv-atomic.ts | 3 ++ src/packages/server/nats/auth.ts | 37 +++++++++++++++---- src/packages/sync/table/changefeed-nats.ts | 2 +- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index add0da192d..c08119e6fa 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -147,6 +147,9 @@ export class SyncTableKVAtomic extends EventEmitter { // watch for new changes async *watch() { + if (this.kv == null) { + throw Error("not initialized"); + } const w = await this.kv.watch({ key: `${this.natsKeyPrefix}.>`, include: "updates", diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 9ab8343a3b..e260599ee3 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -113,22 +113,28 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { const goalPub = new Set([ "_INBOX.>", // so can use request/response `hub.${userType}.${userId}.>`, // can talk as *only this user* to the hub's api's - "$JS.API.>", // so can use Jestream: TODO: too much???! - "$SRV.>", // TODO: obviously vastly too general! - ">", + "$JS.API.INFO", ]); const goalSub = new Set([ "_INBOX.>", // so can user request/response - "$JS.API.>", // TODO! This needs to be restrained more, I think??! Don't know. + //"$JS.API.>", // TODO! This needs to be restrained more, I think??! Don't know. "system.>", // access to READ the system info kv store. - "$SRV.>", // TODO: obviously vastly too general! - ">", ]); if (userType == "account") { goalSub.add(`*.account-${userId}.>`); goalPub.add(`*.account-${userId}.>`); + // microservices api + goalSub.add(`$SRV.*.account-${userId}.>`); + goalSub.add(`$SRV.*.account-${userId}`); + goalSub.add(`$SRV.*`); + goalPub.add(`$SRV.*`); + + // jetstream + goalPub.add(`$JS.API.*.*.KV_account-${userId}`); + goalPub.add(`$JS.API.*.*.KV_account-${userId}.>`); + const pool = getPool(); // all RUNNING projects with the user's group const query = `SELECT project_id, users#>>'{${userId},group}' AS group FROM projects WHERE state#>>'{state}'='running' AND users ? '${userId}' ORDER BY project_id`; @@ -139,6 +145,11 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { goalPub.add(`*.project-${project_id}.>`); goalSub.add(`*.project-${project_id}.>`); + goalPub.add(`$JS.*.*.*.KV_project-${project_id}`); + goalPub.add(`$JS.*.*.*.KV_project-${project_id}.>`); + goalPub.add(`$JS.*.*.*.project-${project_id}-patches`); + goalPub.add(`$JS.*.*.*.project-${project_id}-patches.>`); + goalPub.add(`$JS.*.*.*.*.project-${project_id}-patches.>`); } // TODO: there will be other subjects // TODO: something similar for projects, e.g., they can publish to a channel that browser clients @@ -150,6 +161,16 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { goalPub.add(`*.project-${userId}.>`); goalSub.add(`*.project-${userId}.>`); + + // microservices api + goalSub.add(`$SRV.*.project-${userId}.>`); + goalSub.add(`$SRV.*.project-${userId}`); + goalSub.add(`$SRV.*`); + goalPub.add(`$SRV.*`); + + // jetstream + goalPub.add(`$JS.API.*.*.KV_project-${userId}`); + goalPub.add(`$JS.API.*.*.KV_project-${userId}.>`); } // **Subject Permissions SYNC Algorithm ** @@ -237,7 +258,7 @@ export async function addProjectPermission({ account_id, project_id }) { "--allow-sub", `project.${project_id}.>,*.project-${project_id}.>`, "--allow-pub", - `project.${project_id}.>,*.project-${project_id}.>`, + `project.${project_id}.>,*.project-${project_id}.>,$JS.*.*.*.KV_project-${project_id},$JS.*.*.*.KV_project-${project_id}.>,$JS.*.*.*.project-${project_id}-patches,$JS.*.*.*.project-${project_id}-patches.>,$JS.*.*.*.*.project-${project_id}-patches.>`, ]); await pushToServer(); } @@ -250,7 +271,7 @@ export async function removeProjectPermission({ account_id, project_id }) { "--sk", name, "--rm", - `project.${project_id}.>,*.project-${project_id}.>`, + `project.${project_id}.>,*.project-${project_id}.>,$JS.*.*.*.KV_project-${project_id},$JS.*.*.*.KV_project-${project_id}.>,$JS.*.*.*.KV_project-${project_id}-patches,$JS.*.*.*.KV_project-${project_id}-patches.>,$JS.*.*.*.*.project-${project_id}-patches.>`, ]); await pushToServer(); } diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index 33befd499e..c540fd4e27 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -33,7 +33,7 @@ export class NatsChangefeed extends EventEmitter { }; close = (): void => { - this.natsSynctable.close(); + this.natsSynctable?.close(); this.state = "closed"; this.emit("close"); }; From 6c8c6b94f8bb7931e8dcd0d1e0412ef92dcd4405 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 6 Feb 2025 03:34:55 +0000 Subject: [PATCH 127/281] nats: fix terminal streams permissions --- .../terminal-editor/nats-terminal-connection.ts | 5 ++--- src/packages/nats/names.ts | 5 +---- src/packages/project/nats/names.ts | 2 +- src/packages/server/nats/auth.ts | 11 ++++++++++- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 667295dfd7..4e2b7299aa 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -11,7 +11,7 @@ const client = uuid(); export class NatsTerminalConnection extends EventEmitter { private project_id: string; - private compute_server_id: number; + //private compute_server_id: number; private path: string; private subject: string; private cmd_subject: string; @@ -44,7 +44,7 @@ export class NatsTerminalConnection extends EventEmitter { }) { super(); this.project_id = project_id; - this.compute_server_id = compute_server_id; + //this.compute_server_id = compute_server_id; this.path = path; this.terminalResize = terminalResize; this.keep = keep; @@ -123,7 +123,6 @@ export class NatsTerminalConnection extends EventEmitter { const { nats_client } = webapp_client; const streamName = projectStreamName({ project_id: this.project_id, - compute_server_id: this.compute_server_id, service: "terminal", }); const nc = await nats_client.getConnection(); diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts index c62e9a085a..cede02c2e2 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/nats/names.ts @@ -48,21 +48,19 @@ export function projectSubject({ export function projectStreamName({ project_id, - compute_server_id = 0, // service = optional name of the microservice, e.g., 'api', 'terminal' service, // path = optional name of specific path for that microservice -- replaced by its sha1 path, }: { project_id: string; - compute_server_id?: number; service?: string; path?: string; }): string { if (!project_id) { throw Error("project_id must be set"); } - let streamName = `project-${project_id}-${compute_server_id}`; + let streamName = `project-${project_id}`; if (service) { streamName += "-" + service; if (path) { @@ -72,7 +70,6 @@ export function projectStreamName({ return streamName; } - export function browserSubject({ account_id, sessionId, service }) { if (!sessionId) { throw Error("sessionId must be set"); diff --git a/src/packages/project/nats/names.ts b/src/packages/project/nats/names.ts index 9afaf5475c..dcca25bdc8 100644 --- a/src/packages/project/nats/names.ts +++ b/src/packages/project/nats/names.ts @@ -6,5 +6,5 @@ export function getSubject(opts: { path?; service? }) { } export function getStreamName(opts: { path?; service? }) { - return projectStreamName({ ...opts, compute_server_id, project_id }); + return projectStreamName({ ...opts, project_id }); } diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index e260599ee3..f65086f4c4 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -150,6 +150,9 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { goalPub.add(`$JS.*.*.*.project-${project_id}-patches`); goalPub.add(`$JS.*.*.*.project-${project_id}-patches.>`); goalPub.add(`$JS.*.*.*.*.project-${project_id}-patches.>`); + goalPub.add(`$JS.*.*.*.project-${project_id}-terminal`); + goalPub.add(`$JS.*.*.*.project-${project_id}-terminal.>`); + goalPub.add(`$JS.*.*.*.*.project-${project_id}-terminal.>`); } // TODO: there will be other subjects // TODO: something similar for projects, e.g., they can publish to a channel that browser clients @@ -171,6 +174,12 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { // jetstream goalPub.add(`$JS.API.*.*.KV_project-${userId}`); goalPub.add(`$JS.API.*.*.KV_project-${userId}.>`); + goalPub.add(`$JS.*.*.*.project-${userId}-patches`); + goalPub.add(`$JS.*.*.*.project-${userId}-patches.>`); + goalPub.add(`$JS.*.*.*.*.project-${userId}-patches.>`); + goalPub.add(`$JS.*.*.*.project-${userId}-terminal`); + goalPub.add(`$JS.*.*.*.project-${userId}-terminal.>`); + goalPub.add(`$JS.*.*.*.*.project-${userId}-terminal.>`); } // **Subject Permissions SYNC Algorithm ** @@ -258,7 +267,7 @@ export async function addProjectPermission({ account_id, project_id }) { "--allow-sub", `project.${project_id}.>,*.project-${project_id}.>`, "--allow-pub", - `project.${project_id}.>,*.project-${project_id}.>,$JS.*.*.*.KV_project-${project_id},$JS.*.*.*.KV_project-${project_id}.>,$JS.*.*.*.project-${project_id}-patches,$JS.*.*.*.project-${project_id}-patches.>,$JS.*.*.*.*.project-${project_id}-patches.>`, + `project.${project_id}.>,*.project-${project_id}.>,$JS.*.*.*.KV_project-${project_id},$JS.*.*.*.KV_project-${project_id}.>,$JS.*.*.*.project-${project_id}-patches,$JS.*.*.*.project-${project_id}-patches.>,$JS.*.*.*.*.project-${project_id}-patches.>,$JS.*.*.*.project-${project_id}-terminal,$JS.*.*.*.project-${project_id}-terminal.>,$JS.*.*.*.*.project-${project_id}-terminal.>`, ]); await pushToServer(); } From 743f46dba73a0f7679a25e8fb2aa371c8b13902d Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 6 Feb 2025 03:48:32 +0000 Subject: [PATCH 128/281] nats auth: get rid of redundant code for specifying rules --- src/packages/server/nats/auth.ts | 80 ++++++++++++++++---------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index f65086f4c4..cbf766b151 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -139,47 +139,24 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { // all RUNNING projects with the user's group const query = `SELECT project_id, users#>>'{${userId},group}' AS group FROM projects WHERE state#>>'{state}'='running' AND users ? '${userId}' ORDER BY project_id`; const { rows } = await pool.query(query); - for (const { project_id /*, group */ } of rows) { - goalPub.add(`project.${project_id}.>`); - goalSub.add(`project.${project_id}.>`); - - goalPub.add(`*.project-${project_id}.>`); - goalSub.add(`*.project-${project_id}.>`); - goalPub.add(`$JS.*.*.*.KV_project-${project_id}`); - goalPub.add(`$JS.*.*.*.KV_project-${project_id}.>`); - goalPub.add(`$JS.*.*.*.project-${project_id}-patches`); - goalPub.add(`$JS.*.*.*.project-${project_id}-patches.>`); - goalPub.add(`$JS.*.*.*.*.project-${project_id}-patches.>`); - goalPub.add(`$JS.*.*.*.project-${project_id}-terminal`); - goalPub.add(`$JS.*.*.*.project-${project_id}-terminal.>`); - goalPub.add(`$JS.*.*.*.*.project-${project_id}-terminal.>`); + for (const { project_id } of rows) { + const { pub, sub } = projectSubjects(project_id); + add(goalSub, sub); + add(goalPub, pub); } // TODO: there will be other subjects // TODO: something similar for projects, e.g., they can publish to a channel that browser clients // will listen to, e.g., for timetravel editing. } else if (userType == "project") { - // the project can publish to anything under its own subject: - goalPub.add(`project.${userId}.>`); - goalSub.add(`project.${userId}.>`); - - goalPub.add(`*.project-${userId}.>`); - goalSub.add(`*.project-${userId}.>`); - // microservices api goalSub.add(`$SRV.*.project-${userId}.>`); goalSub.add(`$SRV.*.project-${userId}`); goalSub.add(`$SRV.*`); goalPub.add(`$SRV.*`); - // jetstream - goalPub.add(`$JS.API.*.*.KV_project-${userId}`); - goalPub.add(`$JS.API.*.*.KV_project-${userId}.>`); - goalPub.add(`$JS.*.*.*.project-${userId}-patches`); - goalPub.add(`$JS.*.*.*.project-${userId}-patches.>`); - goalPub.add(`$JS.*.*.*.*.project-${userId}-patches.>`); - goalPub.add(`$JS.*.*.*.project-${userId}-terminal`); - goalPub.add(`$JS.*.*.*.project-${userId}-terminal.>`); - goalPub.add(`$JS.*.*.*.*.project-${userId}-terminal.>`); + const { pub, sub } = projectSubjects(userId); + add(goalSub, sub); + add(goalPub, pub); } // **Subject Permissions SYNC Algorithm ** @@ -259,28 +236,31 @@ export async function addProjectPermission({ account_id, project_id }) { throw Error("user must be collaborator on project"); } const name = getNatsUserName({ account_id }); + const { pub, sub } = projectSubjects(project_id); await nsc([ "edit", "signing-key", "--sk", name, "--allow-sub", - `project.${project_id}.>,*.project-${project_id}.>`, + Array.from(sub).join(","), "--allow-pub", - `project.${project_id}.>,*.project-${project_id}.>,$JS.*.*.*.KV_project-${project_id},$JS.*.*.*.KV_project-${project_id}.>,$JS.*.*.*.project-${project_id}-patches,$JS.*.*.*.project-${project_id}-patches.>,$JS.*.*.*.*.project-${project_id}-patches.>,$JS.*.*.*.project-${project_id}-terminal,$JS.*.*.*.project-${project_id}-terminal.>,$JS.*.*.*.*.project-${project_id}-terminal.>`, + Array.from(pub).join(","), ]); await pushToServer(); } export async function removeProjectPermission({ account_id, project_id }) { const name = getNatsUserName({ account_id }); + const { pub, sub } = projectSubjects(project_id); + add(pub, sub); await nsc([ "edit", "signing-key", "--sk", name, "--rm", - `project.${project_id}.>,*.project-${project_id}.>,$JS.*.*.*.KV_project-${project_id},$JS.*.*.*.KV_project-${project_id}.>,$JS.*.*.*.KV_project-${project_id}-patches,$JS.*.*.*.KV_project-${project_id}-patches.>,$JS.*.*.*.*.project-${project_id}-patches.>`, + Array.from(pub).join(","), ]); await pushToServer(); } @@ -349,14 +329,6 @@ export async function createNatsUser(cocalcUser: CoCalcUser) { pushToServer(); } -// export async function updateActiveCollaborators(project_id: string) { -// const pool = getPool(); -// const { rows } = await pool.query( -// "select account_id from accounts where account_id=any(select jsonb_object_keys(users)::uuid from projects where project_id=$1) and last_active >= now() - interval '1 day'", -// [project_id], -// ); -// } - export async function getJwt(cocalcUser: CoCalcUser): Promise { try { return await getNatsUserJwt(cocalcUser); @@ -365,3 +337,29 @@ export async function getJwt(cocalcUser: CoCalcUser): Promise { return await getNatsUserJwt(cocalcUser); } } + +function projectSubjects(project_id: string) { + const pub = new Set([]); + const sub = new Set([]); + pub.add(`project.${project_id}.>`); + sub.add(`project.${project_id}.>`); + + pub.add(`*.project-${project_id}.>`); + sub.add(`*.project-${project_id}.>`); + + pub.add(`$JS.*.*.*.KV_project-${project_id}`); + pub.add(`$JS.*.*.*.KV_project-${project_id}.>`); + + for (const name of ["patches", "terminal"]) { + pub.add(`$JS.*.*.*.project-${project_id}-${name}`); + pub.add(`$JS.*.*.*.project-${project_id}-${name}.>`); + pub.add(`$JS.*.*.*.*.project-${project_id}-${name}.>`); + } + return { pub, sub }; +} + +function add(X: Set, Y: Set) { + for (const y of Y) { + X.add(y); + } +} From e17448cbe0a7e03d4579a57c3f934e83ef41d256 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 6 Feb 2025 20:43:07 +0000 Subject: [PATCH 129/281] account prefs save error crashed server so fix this and make error better --- src/packages/frontend/account/table-error.tsx | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/packages/frontend/account/table-error.tsx b/src/packages/frontend/account/table-error.tsx index f7dacbd491..9aa982e051 100644 --- a/src/packages/frontend/account/table-error.tsx +++ b/src/packages/frontend/account/table-error.tsx @@ -3,8 +3,8 @@ Show an error if something goes wrong trying to save the account settings table to the database. */ -import { Alert } from "antd"; -import { useTypedRedux } from "../app-framework"; +import ShowError from "@cocalc/frontend/components/error"; +import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; export default function AccountTableError() { const tableError = useTypedRedux("account", "tableError"); @@ -22,30 +22,29 @@ export default function AccountTableError() { } let description; - if (obj["name"] != null) { + if (obj?.["name"] != null) { // Issue trying to set the username. description = "Please try a different username. Names can be between 1 and 39 characters, contain upper and lower case letters, numbers, and dashes."; } else { - description = ( - <> - There was an error trying to save an account setting to the server. In - particular, the following change failed: - 
-          {JSON.stringify(obj, undefined, 2)}
-
- Try changing the relevant field below. - - ); + description = ` +There was an error trying to save an account setting to the server. In +particular, the following change failed: + +\`\`\`js +${JSON.stringify(obj, undefined, 2)} +\`\`\` +`; } return (
- + redux.getActions("account").setState({ tableError: undefined }) + } style={{ margin: "15px auto", maxWidth: "900px" }} - message={{error}} - description={description} - type="error" />
); From 3e8bb4462612fa07e978ce7feef6ddb11f6018dd Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 6 Feb 2025 20:43:29 +0000 Subject: [PATCH 130/281] nats: unify and improve the default connection parameters --- src/packages/backend/nats/index.ts | 4 ++-- src/packages/frontend/nats/client.ts | 16 +++++++--------- src/packages/nats/sync/synctable-stream.ts | 2 -- src/packages/next/pages/api/v2/auth/sign-in.ts | 8 -------- src/packages/project/nats/connection.ts | 2 ++ src/packages/util/nats.ts | 12 +++++++++++- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/packages/backend/nats/index.ts b/src/packages/backend/nats/index.ts index ecc992433a..8c1bec851e 100644 --- a/src/packages/backend/nats/index.ts +++ b/src/packages/backend/nats/index.ts @@ -6,6 +6,7 @@ import { connect, credsAuthenticator } from "nats"; export { getEnv } from "./env"; import { delay } from "awaiting"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { CONNECT_OPTIONS } from "@cocalc/util/nats"; const logger = getLogger("backend:nats"); @@ -30,9 +31,8 @@ export const getConnection = reuseInFlight(async () => { try { const creds = await getCreds(); nc = await connect({ + ...CONNECT_OPTIONS, authenticator: credsAuthenticator(new TextEncoder().encode(creds)), - // bound on how long after network or server goes down until starts working again - pingInterval: 10000, }); logger.debug(`connected to ${nc.getServer()}`); } catch (err) { diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index b965642145..a768b5a7a5 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -23,6 +23,7 @@ import { KV } from "@cocalc/nats/sync/kv"; import { initApi } from "@cocalc/frontend/nats/api"; import { delay } from "awaiting"; import { Svcm } from "@nats-io/services"; +import { CONNECT_OPTIONS } from "@cocalc/util/nats"; export class NatsClient { client: WebappClient; @@ -64,19 +65,16 @@ export class NatsClient { } const server = `${location.protocol == "https:" ? "wss" : "ws"}://${location.host}${appBasePath}/nats`; console.log(`NATS: connecting to ${server}...`); + const options = { + ...CONNECT_OPTIONS, + servers: [server], + }; try { - this.nc = await nats.connect({ - servers: [server], - // this pingInterval determines how long from when the browser's network connection dies - // and comes back, until nats starts working again. - pingInterval: 10000, - }); + this.nc = await nats.connect(options); } catch (err) { console.log("NATS: set the JWT cookie and try again"); await fetch(join(appBasePath, "nats")); - this.nc = await nats.connect({ - servers: [server], - }); + this.nc = await nats.connect(options); } console.log(`NATS: connected to ${server}`); return this.nc; diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts index eb36fa447f..909cc69399 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/nats/sync/synctable-stream.ts @@ -5,8 +5,6 @@ for streaming data. This is ONLY for the scope of patches in a single project. It uses a NATS stream to store the elements in a well defined order. - - */ import { jetstreamManager, jetstream } from "@nats-io/jetstream"; diff --git a/src/packages/next/pages/api/v2/auth/sign-in.ts b/src/packages/next/pages/api/v2/auth/sign-in.ts index ad63587eca..7ae184b7f0 100644 --- a/src/packages/next/pages/api/v2/auth/sign-in.ts +++ b/src/packages/next/pages/api/v2/auth/sign-in.ts @@ -17,18 +17,10 @@ Sign in works as follows: */ import getPool from "@cocalc/database/pool"; -import { createRememberMeCookie } from "@cocalc/server/auth/remember-me"; -import { - ACCOUNT_ID_COOKIE_NAME, - NATS_JWT_COOKIE_NAME, - REMEMBER_ME_COOKIE_NAME, -} from "@cocalc/backend/auth/cookie-names"; import { recordFail, signInCheck } from "@cocalc/server/auth/throttle"; -import Cookies from "cookies"; import getParams from "lib/api/get-params"; import { verify } from "password-hash"; import { Request, Response } from "express"; -import { getServerSettings } from "@cocalc/database/settings/server-settings"; import setSignInCookies from "@cocalc/server/auth/set-sign-in-cookies"; export default async function signIn(req: Request, res: Response) { diff --git a/src/packages/project/nats/connection.ts b/src/packages/project/nats/connection.ts index 5c2d6bf66f..d1b58c3c6d 100644 --- a/src/packages/project/nats/connection.ts +++ b/src/packages/project/nats/connection.ts @@ -1,5 +1,6 @@ import { getLogger } from "@cocalc/project/logger"; import { connect, jwtAuthenticator } from "nats"; +import { CONNECT_OPTIONS } from "@cocalc/util/nats"; const logger = getLogger("project:nats:connection"); @@ -11,6 +12,7 @@ export default async function getConnection() { throw Error("environment variable COCALC_NATS_JWT *must* be set"); } nc = await connect({ + ...CONNECT_OPTIONS, authenticator: jwtAuthenticator(process.env.COCALC_NATS_JWT), }); logger.debug(`connected to ${nc.getServer()}`); diff --git a/src/packages/util/nats.ts b/src/packages/util/nats.ts index e80be8fc26..b6d6b23884 100644 --- a/src/packages/util/nats.ts +++ b/src/packages/util/nats.ts @@ -1,4 +1,14 @@ // Some very generic nats related parameters -// how frequently +// how frequently export const NATS_OPEN_FILE_TOUCH_INTERVAL = 30000; + +export const CONNECT_OPTIONS = { + // this pingInterval determines how long (worse case) from when the connection dies + // and comes back, until nats starts working again. + pingInterval: 10000, + // never give up attempting to reconnect. The default is 10 attempts, but if we allow for + // giving up, then we have to write logic throughout our code to do basically the same + // thing as this, but worse. + maxReconnectAttempts: -1, +} as const; From 97701627ad459cdc7c0c4b559d0b44ba4aaf328b Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 6 Feb 2025 23:57:18 +0000 Subject: [PATCH 131/281] nats: trying a little "eventually consistent KV" POC --- .../nats/sync/eventually-consistent-kv.ts | 98 +++++++++++++++++++ src/packages/nats/sync/kv.ts | 7 +- 2 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 src/packages/nats/sync/eventually-consistent-kv.ts diff --git a/src/packages/nats/sync/eventually-consistent-kv.ts b/src/packages/nats/sync/eventually-consistent-kv.ts new file mode 100644 index 0000000000..adcb606b7f --- /dev/null +++ b/src/packages/nats/sync/eventually-consistent-kv.ts @@ -0,0 +1,98 @@ +/* +Eventually Consistent Distributed Key:Value Store + +DEVELOPMENT: + +~/cocalc/src/packages/server$ node +Welcome to Node.js v18.17.1. +Type ".help" for more information. +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/eventually-consistent-kv"); s = new a.EventuallyConsistentKV({name:'test',env,filter:['foo.>'],merge:({parent,local,remote})=>local}); await s.init(); +*/ + +import { EventEmitter } from "events"; +import { KV } from "./kv"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { type NatsEnv } from "@cocalc/nats/types"; + +const TOMBSTONE = Symbol("tombstone"); + +export class EventuallyConsistentKV extends EventEmitter { + private kv: KV; + private local: { [key: string]: any } = {}; + private merge: (opts: { parent; local; remote }) => any; + + constructor({ + name, + env, + filter, + merge, + options, + }: { + name: string; + env: NatsEnv; + // conflict resolution + merge: (opts: { parent; local; remote }) => any; + // filter: optionally restrict to subset of named kv store matching these subjects. + // NOTE: any key name that you *set or delete* should match one of these + filter?: string | string[]; + options?; + }) { + super(); + this.merge = merge; + this.kv = new KV({ name, env, filter, options }); + return new Proxy(this, { + set(target, prop, value) { + if (!target.kv.isValidKey(String(prop))) { + throw Error(`set: key (=${String(prop)}) must match the filter`); + } + target.set(prop, value); + return true; + }, + get(target, prop) { + const x = + target[prop] ?? target.local[String(prop)] ?? target.kv.get(prop); + return x === TOMBSTONE ? undefined : x; + }, + }); + } + + init = reuseInFlight(async () => { + await this.kv.init(); + }); + + get = () => { + const x = { ...this.kv.get(), ...this.local }; + for (const key in this.local) { + if (this.local[key] === TOMBSTONE) { + delete x[key]; + } + } + return x; + }; + + set = (...args) => { + if (args.length == 2) { + this.local[args[0]] = args[1] ?? TOMBSTONE; + return; + } + const obj = args[0]; + for (const key in obj) { + this.local[key] = obj[key] ?? TOMBSTONE; + } + }; + + save = async () => { + const obj = { ...this.local }; + for (const key in obj) { + if (obj[key] === TOMBSTONE) { + obj[key] = undefined; + } + } + await this.kv.set(obj); + for (const key in obj) { + if (obj[key] === this.local[key]) { + delete this.local[key]; + } + } + }; +} diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index 6b154870a4..d83fb16a6b 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -198,7 +198,8 @@ export class KV extends EventEmitter { return this.times?.[key]; } }; - private matches = (key: string) => { + + isValidKey = (key: string) => { if (this.filter == null) { return true; } @@ -211,7 +212,7 @@ export class KV extends EventEmitter { }; delete = async (key, revision?) => { - if (!this.matches(key)) { + if (!this.isValidKey(key)) { throw Error( `delete: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, ); @@ -289,7 +290,7 @@ export class KV extends EventEmitter { }; private setOne = async (key, value) => { - if (!this.matches(key)) { + if (!this.isValidKey(key)) { throw Error( `set: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, ); From f4579bc880c86332f6e97bb3a77f2907b3620653 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 7 Feb 2025 00:44:44 +0000 Subject: [PATCH 132/281] nats kv: work in progress --- .../nats/sync/eventually-consistent-kv.ts | 79 ++++++++++++------- src/packages/nats/sync/kv.ts | 24 +++--- 2 files changed, 64 insertions(+), 39 deletions(-) diff --git a/src/packages/nats/sync/eventually-consistent-kv.ts b/src/packages/nats/sync/eventually-consistent-kv.ts index adcb606b7f..1e6252b102 100644 --- a/src/packages/nats/sync/eventually-consistent-kv.ts +++ b/src/packages/nats/sync/eventually-consistent-kv.ts @@ -6,60 +6,61 @@ DEVELOPMENT: ~/cocalc/src/packages/server$ node Welcome to Node.js v18.17.1. Type ".help" for more information. -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/eventually-consistent-kv"); s = new a.EventuallyConsistentKV({name:'test',env,filter:['foo.>'],merge:({parent,local,remote})=>local}); await s.init(); +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/eventually-consistent-kv"); s = new a.EventuallyConsistentKV({name:'test',env,filter:['foo.>'],resolve:({parent,local,remote})=>{return {...remote,...local}}}); await s.init(); */ import { EventEmitter } from "events"; import { KV } from "./kv"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { type NatsEnv } from "@cocalc/nats/types"; +import { isEqual } from "lodash"; const TOMBSTONE = Symbol("tombstone"); export class EventuallyConsistentKV extends EventEmitter { private kv: KV; private local: { [key: string]: any } = {}; - private merge: (opts: { parent; local; remote }) => any; + private resolve: (opts: { ancestor; local; remote }) => any; + private changed: Set = new Set(); constructor({ name, env, filter, - merge, + resolve, options, }: { name: string; env: NatsEnv; // conflict resolution - merge: (opts: { parent; local; remote }) => any; + resolve: (opts: { ancestor; local; remote }) => any; // filter: optionally restrict to subset of named kv store matching these subjects. // NOTE: any key name that you *set or delete* should match one of these filter?: string | string[]; options?; }) { super(); - this.merge = merge; + this.resolve = resolve; this.kv = new KV({ name, env, filter, options }); - return new Proxy(this, { - set(target, prop, value) { - if (!target.kv.isValidKey(String(prop))) { - throw Error(`set: key (=${String(prop)}) must match the filter`); - } - target.set(prop, value); - return true; - }, - get(target, prop) { - const x = - target[prop] ?? target.local[String(prop)] ?? target.kv.get(prop); - return x === TOMBSTONE ? undefined : x; - }, - }); } init = reuseInFlight(async () => { + this.kv.on("change", this.handleRemoteChange); await this.kv.init(); }); + private handleRemoteChange = (key, remote, ancestor) => { + const local = this.local[key]; + if (local !== undefined) { + const value = this.resolve({ local, remote, ancestor }); + if (isEqual(value, remote)) { + delete this.local[key]; + } else { + this.local[key] = value ?? TOMBSTONE; + } + } + }; + get = () => { const x = { ...this.kv.get(), ...this.local }; for (const key in this.local) { @@ -70,29 +71,53 @@ export class EventuallyConsistentKV extends EventEmitter { return x; }; + delete = (key) => { + this.local[key] = TOMBSTONE; + this.changed.add(key); + }; + set = (...args) => { if (args.length == 2) { this.local[args[0]] = args[1] ?? TOMBSTONE; - return; + this.changed.add(args[0]); + } else { + const obj = args[0]; + for (const key in obj) { + this.local[key] = obj[key] ?? TOMBSTONE; + this.changed.add(key); + } } - const obj = args[0]; - for (const key in obj) { - this.local[key] = obj[key] ?? TOMBSTONE; + this.tryToSave(); + }; + + private tryToSave = async () => { + try { + await this.save(); + } catch (err) { + console.log("problem saving", err); + } + if (Object.keys(this.local).length > 0) { + setTimeout(this.tryToSave, 100); } }; - save = async () => { + private save = reuseInFlight(async () => { + this.changed.clear(); const obj = { ...this.local }; for (const key in obj) { if (obj[key] === TOMBSTONE) { - obj[key] = undefined; + await this.kv.delete(key); + delete obj[key]; + if (!this.changed.has(key)) { + delete this.local[key]; + } } } await this.kv.set(obj); for (const key in obj) { - if (obj[key] === this.local[key]) { + if (obj[key] === this.local[key] && !this.changed.has(key)) { delete this.local[key]; } } - }; + }); } diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index d83fb16a6b..78034ced10 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -110,15 +110,15 @@ export class KV extends EventEmitter { : typeof filter == "string" ? [filter] : filter; - return new Proxy(this, { - set(target, prop, value) { - target.setOne(prop, value); - return true; - }, - get(target, prop) { - return target[prop] ?? target.all?.[String(prop)]; - }, - }); + // return new Proxy(this, { + // set(target, prop, value) { + // target.setOne(prop, value); + // return true; + // }, + // get(target, prop) { + // return target[prop] ?? target.all?.[String(prop)]; + // }, + // }); } init = reuseInFlight(async () => { @@ -147,8 +147,7 @@ export class KV extends EventEmitter { private startWatch = async () => { // watch for changes this.watch = await this.kv.watch({ - // we assume that we ONLY delete old items which are not relevant - ignoreDeletes: true, + ignoreDeletes: false, include: "updates", key: this.filter, }); @@ -159,6 +158,7 @@ export class KV extends EventEmitter { return; } this.revisions[key] = revision; + const prev = this.all[key]; if (value.length == 0) { // delete delete this.all[key]; @@ -167,7 +167,7 @@ export class KV extends EventEmitter { this.all[key] = this.env.jc.decode(value); this.times[key] = sm.time; } - this.emit("change", key, this.all[key]); + this.emit("change", key, this.all[key], prev); } }; From 80415137475c9f5b23b5fd14047355bc979f8f23 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 7 Feb 2025 02:18:30 +0000 Subject: [PATCH 133/281] nats: more work on eventually consistent key value store --- src/packages/frontend/nats/client.ts | 38 +++++++- src/packages/nats/names.ts | 19 ++++ .../nats/sync/eventually-consistent-kv.ts | 90 ++++++++++++++++--- src/packages/nats/sync/kv.ts | 32 +++---- 4 files changed, 143 insertions(+), 36 deletions(-) diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index a768b5a7a5..1b05c4ad11 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -6,7 +6,7 @@ import { join } from "path"; import * as jetstream from "@nats-io/jetstream"; import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; import { randomId } from "@cocalc/nats/names"; -import { browserSubject, projectSubject } from "@cocalc/nats/names"; +import { browserSubject, projectSubject, kvName } from "@cocalc/nats/names"; import { parse_query } from "@cocalc/sync/table/util"; import { sha1 } from "@cocalc/util/misc"; import { keys } from "lodash"; @@ -20,6 +20,7 @@ import { PubSub } from "@cocalc/nats/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; import { SystemKv } from "@cocalc/nats/system"; import { KV } from "@cocalc/nats/sync/kv"; +import { EventuallyConsistentKV } from "@cocalc/nats/sync/eventually-consistent-kv"; import { initApi } from "@cocalc/frontend/nats/api"; import { delay } from "awaiting"; import { Svcm } from "@nats-io/services"; @@ -394,14 +395,17 @@ export class NatsClient { private kvCache: { [key: string]: KV } = {}; kv = reuseInFlight( async ({ - name, + account_id, + project_id, filter, options, }: { - name: string; + account_id?: string; + project_id?: string; filter?: string | string[]; options?; }) => { + const name = kvName({ account_id, project_id }); const key = JSON.stringify([name, filter, options]); if (this.kvCache[key] == null) { const kv = new KV({ @@ -420,6 +424,34 @@ export class NatsClient { }, ); + eckv = async ({ + account_id, + project_id, + filter, + options, + resolve, + }: { + account_id?: string; + project_id?: string; + filter?: string | string[]; + options?; + resolve: (opts: { ancestor; local; remote }) => any; + }) => { + if (!account_id && !project_id) { + account_id = this.client.account_id; + } + const name = kvName({ account_id, project_id }); + const eckv = new EventuallyConsistentKV({ + name, + filter, + options, + resolve, + env: await this.getEnv(), + }); + await eckv.init(); + return eckv; + }; + microservicesClient = async () => { const nc = await this.getConnection(); // @ts-ignore diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts index cede02c2e2..909fd520c6 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/nats/names.ts @@ -20,6 +20,25 @@ export function randomId() { return generateVouchers({ count: 1, length: 10 })[0]; } +export function kvName({ + project_id, + account_id, +}: { + project_id?: string; + account_id?: string; +}) { + if (project_id) { + if (account_id) { + throw Error("both account_id and project_id can't be set"); + } + return `project-${project_id}`; + } + if (!account_id) { + throw Error("at least one of account_id and project_id must be set"); + } + return `account-${account_id}`; +} + export function projectSubject({ project_id, compute_server_id = 0, diff --git a/src/packages/nats/sync/eventually-consistent-kv.ts b/src/packages/nats/sync/eventually-consistent-kv.ts index 1e6252b102..5dc073df4a 100644 --- a/src/packages/nats/sync/eventually-consistent-kv.ts +++ b/src/packages/nats/sync/eventually-consistent-kv.ts @@ -7,6 +7,18 @@ DEVELOPMENT: Welcome to Node.js v18.17.1. Type ".help" for more information. > env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/eventually-consistent-kv"); s = new a.EventuallyConsistentKV({name:'test',env,filter:['foo.>'],resolve:({parent,local,remote})=>{return {...remote,...local}}}); await s.init(); + + +In the browser console: + +> s = await cc.client.nats_client.eckv({filter:['foo.>'],resolve:({parent,local,remote})=>{return {...remote,...local}}}) + +# NOTE that the name is account-{account_id} or project-{project_id}, +# and if not given defaults to the account-{user's account id} +> s.kv.name +'account-6aae57c6-08f1-4bb5-848b-3ceb53e61ede' + +> s.on('change',(key)=>console.log(key));0; */ import { EventEmitter } from "events"; @@ -14,11 +26,12 @@ import { KV } from "./kv"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { type NatsEnv } from "@cocalc/nats/types"; import { isEqual } from "lodash"; +import { delay } from "awaiting"; const TOMBSTONE = Symbol("tombstone"); export class EventuallyConsistentKV extends EventEmitter { - private kv: KV; + private kv?: KV; private local: { [key: string]: any } = {}; private resolve: (opts: { ancestor; local; remote }) => any; private changed: Set = new Set(); @@ -45,10 +58,24 @@ export class EventuallyConsistentKV extends EventEmitter { } init = reuseInFlight(async () => { + if (this.kv == null) { + throw Error("closed"); + } this.kv.on("change", this.handleRemoteChange); await this.kv.init(); + this.emit("connected"); }); + close = () => { + if (this.kv == null) { + return; + } + this.kv.close(); + this.emit("closed"); + this.removeAllListeners(); + delete this.kv; + }; + private handleRemoteChange = (key, remote, ancestor) => { const local = this.local[key]; if (local !== undefined) { @@ -59,9 +86,21 @@ export class EventuallyConsistentKV extends EventEmitter { this.local[key] = value ?? TOMBSTONE; } } + this.emit("change", key); }; - get = () => { + get = (key?) => { + if (this.kv == null) { + throw Error("closed"); + } + if (key != null) { + this.assertValidKey(key); + const local = this.local[key]; + if (local === TOMBSTONE) { + return undefined; + } + return local ?? this.kv.get(key); + } const x = { ...this.kv.get(), ...this.local }; for (const key in this.local) { if (this.local[key] === TOMBSTONE) { @@ -71,37 +110,62 @@ export class EventuallyConsistentKV extends EventEmitter { return x; }; + private assertValidKey = (key) => { + if (this.kv == null) { + throw Error("closed"); + } + this.kv.assertValidKey(key); + }; + delete = (key) => { + this.assertValidKey(key); this.local[key] = TOMBSTONE; this.changed.add(key); + this.save(); }; set = (...args) => { if (args.length == 2) { + this.assertValidKey(args[0]); this.local[args[0]] = args[1] ?? TOMBSTONE; this.changed.add(args[0]); } else { const obj = args[0]; for (const key in obj) { + this.assertValidKey(key); this.local[key] = obj[key] ?? TOMBSTONE; this.changed.add(key); } } - this.tryToSave(); + this.save(); }; - private tryToSave = async () => { - try { - await this.save(); - } catch (err) { - console.log("problem saving", err); - } - if (Object.keys(this.local).length > 0) { - setTimeout(this.tryToSave, 100); - } - }; + hasUnsavedChanges = () => + this.changed.size > 0 || Object.keys(this.local).length > 0; private save = reuseInFlight(async () => { + let d = 100; + while (true) { + try { + await this.attemptToSave(); + //console.log("successfully saved"); + } catch { + //(err) { + // console.log("problem saving", err); + } + if (this.hasUnsavedChanges()) { + d = Math.min(10000, d * 1.3) + Math.random() * 100; + await delay(d); + } else { + return; + } + } + }); + + private attemptToSave = reuseInFlight(async () => { + if (this.kv == null) { + throw Error("closed"); + } this.changed.clear(); const obj = { ...this.local }; for (const key in obj) { diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index 78034ced10..a8ec3d8d3f 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -199,6 +199,14 @@ export class KV extends EventEmitter { } }; + assertValidKey = (key: string) => { + if (!this.isValidKey(key)) { + throw Error( + `delete: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, + ); + } + }; + isValidKey = (key: string) => { if (this.filter == null) { return true; @@ -212,11 +220,7 @@ export class KV extends EventEmitter { }; delete = async (key, revision?) => { - if (!this.isValidKey(key)) { - throw Error( - `delete: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, - ); - } + this.assertValidKey(key); if (this.all == null || this.revisions == null || this.times == null) { throw Error("not ready"); } @@ -306,20 +310,8 @@ export class KV extends EventEmitter { } const revision = this.revisions[key]; const val = this.env.jc.encode(value); - const cur = this.all[key]; - try { - this.all[key] = value; - const newRevision = await this.kv.put(key, val, { - previousSeq: revision, - }); - this.revisions[key] = newRevision; - } catch (err) { - if (cur === undefined) { - delete this.all[key]; - } else { - this.all[key] = cur; - } - throw err; - } + await this.kv.put(key, val, { + previousSeq: revision, + }); }; } From b28442e48c9431f837a376993adb3063596f347a Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 7 Feb 2025 03:09:54 +0000 Subject: [PATCH 134/281] nats distributed key value store -- rename and document --- src/packages/frontend/nats/client.ts | 23 ++-- .../{eventually-consistent-kv.ts => dkv.ts} | 106 +++++++++++++++--- src/packages/nats/sync/kv.ts | 56 ++++++--- 3 files changed, 147 insertions(+), 38 deletions(-) rename src/packages/nats/sync/{eventually-consistent-kv.ts => dkv.ts} (51%) diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 1b05c4ad11..19c03724c3 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -20,7 +20,7 @@ import { PubSub } from "@cocalc/nats/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; import { SystemKv } from "@cocalc/nats/system"; import { KV } from "@cocalc/nats/sync/kv"; -import { EventuallyConsistentKV } from "@cocalc/nats/sync/eventually-consistent-kv"; +import { DKV } from "@cocalc/nats/sync/dkv"; import { initApi } from "@cocalc/frontend/nats/api"; import { delay } from "awaiting"; import { Svcm } from "@nats-io/services"; @@ -404,7 +404,10 @@ export class NatsClient { project_id?: string; filter?: string | string[]; options?; - }) => { + } = {}) => { + if (!account_id && !project_id) { + account_id = this.client.account_id; + } const name = kvName({ account_id, project_id }); const key = JSON.stringify([name, filter, options]); if (this.kvCache[key] == null) { @@ -424,32 +427,32 @@ export class NatsClient { }, ); - eckv = async ({ + dkv = async ({ account_id, project_id, filter, options, - resolve, + merge, }: { account_id?: string; project_id?: string; filter?: string | string[]; options?; - resolve: (opts: { ancestor; local; remote }) => any; - }) => { + merge?: (opts: { ancestor; local; remote }) => any; + } = {}) => { if (!account_id && !project_id) { account_id = this.client.account_id; } const name = kvName({ account_id, project_id }); - const eckv = new EventuallyConsistentKV({ + const dkv = new DKV({ name, filter, options, - resolve, + merge, env: await this.getEnv(), }); - await eckv.init(); - return eckv; + await dkv.init(); + return dkv; }; microservicesClient = async () => { diff --git a/src/packages/nats/sync/eventually-consistent-kv.ts b/src/packages/nats/sync/dkv.ts similarity index 51% rename from src/packages/nats/sync/eventually-consistent-kv.ts rename to src/packages/nats/sync/dkv.ts index 5dc073df4a..2dd59a683e 100644 --- a/src/packages/nats/sync/eventually-consistent-kv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -1,17 +1,58 @@ /* Eventually Consistent Distributed Key:Value Store +- You give one or more subjects and this provides a synchronous eventually consistent + "multimaster" distributed way to work with the KV store of keys matching any of those subjects, + inside of the named KV store. +- You should define a 3-way merge function, which is used to automatically resolve all + conflicting writes. The default is to use our local version, i.e., "last write to remote wins". +- All set/get/delete operations are synchronous. +- The state gets sync'd in the backend to NATS as soon as possible. + +This class is based on top of the Consistent Centralized Key:Value Store defined in kv.ts. +You can use the same key:value store at the same time via both interfaces, and if store +is a DKV, you can also access the underlying KV via "store.kv". + +- You must explicitly call "await store.init()" to initialize this before using it. + +- The store emits an event ('change', key) whenever anything changes. + +- Calling "store.get()" provides ALL the data, and "store.get(key)" gets one value. + +- Use "store.set(key,value)" or "store.set({key:value, key2:value2, ...})" to set data, + with the following semantics: + + - in the background, changes propagate to NATS. You do not do anything explicitly and + this should never raise an exception. + + - you can call "store.hasUnsavedChanges()" to see if there are any unsaved changes. + + - call "store.unsavedChanges()" to see the unsaved keys. + +- The 3-way merge function takes as input {local,remote,ancestor,key}, where + - key = the key where there's a conflict + - local = your version of the value + - remote = the remote value, which conflicts in that isEqual(local,remote) is false. + - ancestor = a known common ancestor of local and remote. + + (any of local, remote or ancestor can be undefined, e.g., no previous value or a key was deleted) + + You can do anything synchronously you want to resolve such conflicts, i.e., there are no + axioms that have to be satisifed. If the 3-way merge function throws an exception (or is + not specified) we silently fall back to "last write wins". + + DEVELOPMENT: ~/cocalc/src/packages/server$ node Welcome to Node.js v18.17.1. Type ".help" for more information. -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/eventually-consistent-kv"); s = new a.EventuallyConsistentKV({name:'test',env,filter:['foo.>'],resolve:({parent,local,remote})=>{return {...remote,...local}}}); await s.init(); +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/dkv"); s = new a.DKV({name:'test',env,filter:['foo.>'],merge:({local,remote})=>{return {...remote,...local}}}); await s.init(); In the browser console: -> s = await cc.client.nats_client.eckv({filter:['foo.>'],resolve:({parent,local,remote})=>{return {...remote,...local}}}) +> s = await cc.client.nats_client.dkv({filter:['foo.>'],merge:({local,remote})=>{return {...remote,...local}}}) # NOTE that the name is account-{account_id} or project-{project_id}, # and if not given defaults to the account-{user's account id} @@ -19,6 +60,10 @@ In the browser console: 'account-6aae57c6-08f1-4bb5-848b-3ceb53e61ede' > s.on('change',(key)=>console.log(key));0; + + +TODO: + - require not-everything subject or have an explicit size limit? */ import { EventEmitter } from "events"; @@ -30,30 +75,40 @@ import { delay } from "awaiting"; const TOMBSTONE = Symbol("tombstone"); -export class EventuallyConsistentKV extends EventEmitter { +export class DKV extends EventEmitter { private kv?: KV; + private merge?: (opts: { + key: string; + ancestor: any; + local: any; + remote: any; + }) => any; private local: { [key: string]: any } = {}; - private resolve: (opts: { ancestor; local; remote }) => any; private changed: Set = new Set(); constructor({ name, env, filter, - resolve, + merge, options, }: { name: string; env: NatsEnv; - // conflict resolution - resolve: (opts: { ancestor; local; remote }) => any; + // 3-way merge conflict resolution + merge?: (opts: { + key: string; + ancestor?: any; + local?: any; + remote?: any; + }) => any; // filter: optionally restrict to subset of named kv store matching these subjects. // NOTE: any key name that you *set or delete* should match one of these filter?: string | string[]; options?; }) { super(); - this.resolve = resolve; + this.merge = merge; this.kv = new KV({ name, env, filter, options }); } @@ -74,16 +129,36 @@ export class EventuallyConsistentKV extends EventEmitter { this.emit("closed"); this.removeAllListeners(); delete this.kv; + // @ts-ignore + delete this.local; + // @ts-ignore + delete this.changed; + delete this.merge; }; private handleRemoteChange = (key, remote, ancestor) => { const local = this.local[key]; if (local !== undefined) { - const value = this.resolve({ local, remote, ancestor }); - if (isEqual(value, remote)) { + if (isEqual(local, remote)) { + // we have a local change, but it's the same change as remote, so just + // forget about our local change. delete this.local[key]; } else { - this.local[key] = value ?? TOMBSTONE; + let value; + try { + value = this.merge?.({ key, local, remote, ancestor }); + } catch { + // user provided a merge function that throws an exception. We select local, since + // it is the newest, i.e., "last write wins" + value = local; + } + if (isEqual(value, remote)) { + // no change, so forget our local value + delete this.local[key]; + } else { + // resolve with the new value, or if it is undefined, a TOMBSTONE, meaning choice is to delete. + this.local[key] = value ?? TOMBSTONE; + } } } this.emit("change", key); @@ -140,8 +215,13 @@ export class EventuallyConsistentKV extends EventEmitter { this.save(); }; - hasUnsavedChanges = () => - this.changed.size > 0 || Object.keys(this.local).length > 0; + hasUnsavedChanges = () => { + return this.changed.size > 0 || Object.keys(this.local).length > 0; + }; + + unsavedChanges = () => { + return Object.keys(this.local); + }; private save = reuseInFlight(async () => { let d = 100; diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index a8ec3d8d3f..f257f9fb01 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -1,33 +1,59 @@ /* -This is a simple KV wrapper around NATS's KV, for small KV stores, suitable for configuration data. +Always Consistent Centralized Key Value Store -- it emits an event ('change', key, value) whenever anything changes +- You give one or more subjects and this provides an asynchronous but consistent + way to work with the KV store of keys matching any of those subjects, + inside of the named KV store. +- The get operation is sync. (It can of course be slightly out of date, but that is detected + if you try to immediately write it.) +- The set will fail if the local cached value (returned by get) turns out to be out of date. +- Also delete and set will fail if the NATS connection is down or times out. +- For an eventually consistent sync wrapper around this, use DKV, defined in the sibling file dkv.ts. -- explicitly call "await this.init()" to initialize it +This is a simple KV wrapper around NATS's KV, for small KV stores. Each client holds a local cache +of all data, which is used to ensure set's are a no-op if there is no change. Also, this automates +ensuring that if you do a read-modify-write, this will succeed only if nobody else makes a change +before you. -- calling get() synchronously provides ALL the data. +- You must explicitly call "await store.init()" to initialize it before using it. -- call await set({key:value, key2:value2, ...}) to set data, with the following semantics: +- The store emits an event ('change', key, newValue, previousValue) whenever anything changes - - set ONLY makes a change if our local version (this.get()[key]) of the value is different from - what you're trying to set the value to, where different is defiend by lodash isEqual. +- Calling "store.get()" provides ALL the data and is synchronous. It uses various API tricks to + ensure this is fast and is updated when there is any change from upstream. Use "store.get(key)" + to get the value of one key. - - if our local version this.get()[key] was not the most recent version in NATS, then the set will +- Use "await store.set(key,value)" or "await store.set({key:value, key2:value2, ...})" to set data, + with the following semantics: + + - set ONLY makes a change if our local version ("store.get(key)") of the value is different from + what you're trying to set the value to, where different is defined by lodash isEqual. + + - if our local version this.get(key) was not the most recent version in NATS, then the set will definitely throw an exception! This is fantastic because it means you can modify and save what is in the local cache on multiple nodes at once anywhere, and be 100% certain to never overwrite - data in complicated objects. Of course, you have to assume "await set()" will sometimes fail. + data in complicated objects. Of course, you have to assume "await store.set(...)" WILL + sometimes fail. + + - Set with multiple keys "pipelines" in that MAX_PARALLEL key/value pairs are set at once, without + waiting for every single individual set to get ACK'd from the server before doing more sets. + This makes this **massively** faster, but means that if "await store.set(...)" fails, you don't + immediately know which keys were successfully set and which failed, though all keys worked will get + updated soon and reflected in store.get(). - - set "pipelines" in that MAX_PARALLEL key/value pairs are set at once, without waiting - for each set to get ACK'd from the server before doing more sets. This makes this massively - faster for bigger objects, but means that if "await set({...})" fails, you don't immediately - know which keys were successfully set and which failed, though all that worked will get - updated soon and reflected in get(). +- Use "await store.expire(ageMs)" to delete every key that was last changed at least ageMs + milliseconds in the past. + + TODO/WARNING: the timestamps are defined by NATS (and its clock), but + the definition of "ageMs in the past" is defined by the client where this is called. Thus + if the client's clock is off, that would be a huge problem. An obvious solution is to + get the current time from NATS, and use that. I don't know a "good" way to get the current + time except maybe publishing a message to myself...? TODO: - [ ] maybe expose some functionality related to versions/history? - DEVELOPMENT: ~/cocalc/src/packages/server$ n From eb977e3ebefe9995b8b77291adca3b54bf51770b Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 7 Feb 2025 14:08:11 +0000 Subject: [PATCH 135/281] nats: adding a stream wrapper --- src/packages/frontend/nats/client.ts | 44 +++++++- src/packages/nats/sync/dkv.ts | 4 +- src/packages/nats/sync/kv.ts | 16 +-- src/packages/nats/sync/stream.ts | 158 +++++++++++++++++++++++++++ src/packages/server/nats/auth.ts | 8 +- 5 files changed, 206 insertions(+), 24 deletions(-) create mode 100644 src/packages/nats/sync/stream.ts diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 19c03724c3..7ea5581e3c 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -21,6 +21,7 @@ import type { ChatOptions } from "@cocalc/util/types/llm"; import { SystemKv } from "@cocalc/nats/system"; import { KV } from "@cocalc/nats/sync/kv"; import { DKV } from "@cocalc/nats/sync/dkv"; +import { Stream } from "@cocalc/nats/sync/stream"; import { initApi } from "@cocalc/frontend/nats/api"; import { delay } from "awaiting"; import { Svcm } from "@nats-io/services"; @@ -436,10 +437,10 @@ export class NatsClient { }: { account_id?: string; project_id?: string; - filter?: string | string[]; + filter: string | string[]; options?; - merge?: (opts: { ancestor; local; remote }) => any; - } = {}) => { + merge?: (opts: { ancestor?; local?; remote?; key?: string }) => any; + }) => { if (!account_id && !project_id) { account_id = this.client.account_id; } @@ -455,6 +456,43 @@ export class NatsClient { return dkv; }; + // Browser client gets one stream for each account and one for each project. + // Use the filters to restrict, e.g., to events about a particular file. + private streamCache: { [key: string]: Stream } = {}; + stream = reuseInFlight( + async ({ + account_id, + project_id, + filter, + options, + }: { + account_id?: string; + project_id?: string; + subjects: string | string[]; + filter: string; + options?; + }) => { + const name = kvName({ account_id, project_id }); + const subjects = `${name}.>`; + const key = JSON.stringify([name, subjects, filter, options]); + if (this.streamCache[key] == null) { + const stream = new Stream({ + name, + subjects, + filter, + options, + env: await this.getEnv(), + }); + await stream.init(); + this.streamCache[key] = stream; + stream.on("closed", () => { + delete this.streamCache[key]; + }); + } + return this.streamCache[key]; + }, + ); + microservicesClient = async () => { const nc = await this.getConnection(); // @ts-ignore diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index 2dd59a683e..1486a0b76e 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -103,8 +103,8 @@ export class DKV extends EventEmitter { remote?: any; }) => any; // filter: optionally restrict to subset of named kv store matching these subjects. - // NOTE: any key name that you *set or delete* should match one of these - filter?: string | string[]; + // NOTE: any key name that you *set or delete* must match one of these + filter: string | string[]; options?; }) { super(); diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index f257f9fb01..ed9502feb4 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -130,21 +130,7 @@ export class KV extends EventEmitter { this.env = env; this.name = name; this.options = options; - this.filter = - filter == null - ? undefined - : typeof filter == "string" - ? [filter] - : filter; - // return new Proxy(this, { - // set(target, prop, value) { - // target.setOne(prop, value); - // return true; - // }, - // get(target, prop) { - // return target[prop] ?? target.all?.[String(prop)]; - // }, - // }); + this.filter = typeof filter == "string" ? [filter] : filter; } init = reuseInFlight(async () => { diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts new file mode 100644 index 0000000000..e3e7ac7150 --- /dev/null +++ b/src/packages/nats/sync/stream.ts @@ -0,0 +1,158 @@ +/* +Always Consistent Centralized Stream + +TODO: + - ability to easily initialize with only the + last n messages or only messages going back + to time t + + +DEVELOPMENT: + +~/cocalc/src/packages/server$ n +Welcome to Node.js v18.17.1. +Type ".help" for more information. +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = new a.Stream({name:'test',env,subjects:'foo.>'}); await s.init(); + + +With browser client using a project: + +# in browser +> s = await cc.client.nats_client.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094', filter:'foo'}) + +# in node: +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = new a.Stream({name:'project-56eb622f-d398-489a-83ef-c09f1a1e8094',env,filter:'foo',subjects:'project-56eb622f-d398-489a-83ef-c09f1a1e8094.>'}); await s.init(); + +*/ + +import { EventEmitter } from "events"; +import { type NatsEnv } from "@cocalc/nats/types"; +import { jetstreamManager, jetstream } from "@nats-io/jetstream"; +import { matchesPattern } from "@cocalc/nats/util"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; + +export class Stream extends EventEmitter { + public readonly name: string; + private options?; + private subjects: string | string[]; + private filter?: string; + private env: NatsEnv; + private js; + private stream?; + private consumer?; + private watch?; + private raw: any[] = []; + public readonly events: any[] = []; + + constructor({ + name, + env, + subjects, + filter, + options, + }: { + name: string; + subjects: string | string[]; + filter?: string; + env: NatsEnv; + options?; + }) { + super(); + this.env = env; + // create a jetstream client so we can publish to the stream + this.js = jetstream(env.nc); + this.name = name; + this.options = options; + this.subjects = typeof subjects == "string" ? [subjects] : subjects; + if (this.subjects.length == 0) { + throw Error("subjects must be at least one string"); + } + this.filter = filter; + } + + init = reuseInFlight(async () => { + if (this.stream != null) { + return; + } + const jsm = await jetstreamManager(this.env.nc); + const options = { + subjects: this.subjects, + compression: "s2", + ...this.options, + }; + try { + this.stream = await jsm.streams.add({ + name: this.name, + ...options, + }); + } catch (err) { + // probably already exists, so try to modify to have the requested properties. + this.stream = await jsm.streams.update(this.name, options); + } + this.consumer = await this.getConsumer(); + this.startFetch(); + }); + + publish = async (event, subject?) => { + if (subject != null) { + for (const pattern of this.subjects) { + if (!matchesPattern({ pattern, subject })) { + throw Error( + `subject must match subjects=${JSON.stringify(this.subjects)}`, + ); + } + } + } + subject = subject ?? this.subjects[0]; + if (this.filter) { + if (!matchesPattern({ pattern: this.filter, subject })) { + throw Error(`subject must match filter="${this.filter}"`); + } + } + return await this.js.publish(subject, this.env.jc.encode(event)); + }; + + private getConsumer = async () => { + const js = jetstream(this.env.nc); + const jsm = await jetstreamManager(this.env.nc); + // making an ephemeral consumer + const { name } = await jsm.consumers.add(this.name, { + filter_subject: this.filter, + }); + return await js.consumers.get(this.name, name); + }; + + private startFetch = async () => { + if (this.consumer == null) { + throw Error("consumer not defined"); + } + const consumer = this.consumer; + this.watch = await consumer.fetch(); + for await (const mesg of this.watch) { + this.handle(mesg); + } + }; + + private handle = (mesg) => { + let data; + try { + data = this.env.jc.decode(mesg.data); + } catch { + data = mesg.data; + } + this.events.push(data); + this.raw.push(mesg); + }; + + close = () => { + if (this.watch == null) { + return; + } + this.watch.stop(); + delete this.watch; + delete this.stream; + delete this.consumer; + this.emit("closed"); + this.removeAllListeners(); + }; +} diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index cbf766b151..fa7b18d720 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -350,10 +350,10 @@ function projectSubjects(project_id: string) { pub.add(`$JS.*.*.*.KV_project-${project_id}`); pub.add(`$JS.*.*.*.KV_project-${project_id}.>`); - for (const name of ["patches", "terminal"]) { - pub.add(`$JS.*.*.*.project-${project_id}-${name}`); - pub.add(`$JS.*.*.*.project-${project_id}-${name}.>`); - pub.add(`$JS.*.*.*.*.project-${project_id}-${name}.>`); + for (const name of ["", "-patches", "-terminal"]) { + pub.add(`$JS.*.*.*.project-${project_id}${name}`); + pub.add(`$JS.*.*.*.project-${project_id}${name}.>`); + pub.add(`$JS.*.*.*.*.project-${project_id}${name}.>`); } return { pub, sub }; } From c69423e6db28f7dca7e5b301088e55ef0bc99b62 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 8 Feb 2025 00:22:54 +0000 Subject: [PATCH 136/281] nats stream class: much, much better - building a robust foundation :-) --- src/packages/frontend/nats/client.ts | 51 ++----- src/packages/nats/names.ts | 23 ++- src/packages/nats/sync/kv.ts | 1 - src/packages/nats/sync/stream.ts | 201 +++++++++++++++++++++------ src/packages/nats/util.ts | 10 ++ src/packages/server/nats/auth.ts | 2 +- 6 files changed, 205 insertions(+), 83 deletions(-) diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 7ea5581e3c..2afdbab45f 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -6,7 +6,7 @@ import { join } from "path"; import * as jetstream from "@nats-io/jetstream"; import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; import { randomId } from "@cocalc/nats/names"; -import { browserSubject, projectSubject, kvName } from "@cocalc/nats/names"; +import { browserSubject, projectSubject, jsName } from "@cocalc/nats/names"; import { parse_query } from "@cocalc/sync/table/util"; import { sha1 } from "@cocalc/util/misc"; import { keys } from "lodash"; @@ -21,7 +21,7 @@ import type { ChatOptions } from "@cocalc/util/types/llm"; import { SystemKv } from "@cocalc/nats/system"; import { KV } from "@cocalc/nats/sync/kv"; import { DKV } from "@cocalc/nats/sync/dkv"; -import { Stream } from "@cocalc/nats/sync/stream"; +import { stream } from "@cocalc/nats/sync/stream"; import { initApi } from "@cocalc/frontend/nats/api"; import { delay } from "awaiting"; import { Svcm } from "@nats-io/services"; @@ -409,7 +409,7 @@ export class NatsClient { if (!account_id && !project_id) { account_id = this.client.account_id; } - const name = kvName({ account_id, project_id }); + const name = jsName({ account_id, project_id }); const key = JSON.stringify([name, filter, options]); if (this.kvCache[key] == null) { const kv = new KV({ @@ -444,7 +444,7 @@ export class NatsClient { if (!account_id && !project_id) { account_id = this.client.account_id; } - const name = kvName({ account_id, project_id }); + const name = jsName({ account_id, project_id }); const dkv = new DKV({ name, filter, @@ -456,42 +456,13 @@ export class NatsClient { return dkv; }; - // Browser client gets one stream for each account and one for each project. - // Use the filters to restrict, e.g., to events about a particular file. - private streamCache: { [key: string]: Stream } = {}; - stream = reuseInFlight( - async ({ - account_id, - project_id, - filter, - options, - }: { - account_id?: string; - project_id?: string; - subjects: string | string[]; - filter: string; - options?; - }) => { - const name = kvName({ account_id, project_id }); - const subjects = `${name}.>`; - const key = JSON.stringify([name, subjects, filter, options]); - if (this.streamCache[key] == null) { - const stream = new Stream({ - name, - subjects, - filter, - options, - env: await this.getEnv(), - }); - await stream.init(); - this.streamCache[key] = stream; - stream.on("closed", () => { - delete this.streamCache[key]; - }); - } - return this.streamCache[key]; - }, - ); + stream = async (opts: { + account_id?: string; + project_id?: string; + name: string; + }) => { + return await stream({ ...opts, env: await this.getEnv() }); + }; microservicesClient = async () => { const nc = await this.getConnection(); diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts index 909fd520c6..b0a87456a0 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/nats/names.ts @@ -20,7 +20,9 @@ export function randomId() { return generateVouchers({ count: 1, length: 10 })[0]; } -export function kvName({ +// jetstream name -- we use this canonical name for the KV and the stream associated +// to a project or account. We use the same name for both. +export function jsName({ project_id, account_id, }: { @@ -39,6 +41,25 @@ export function kvName({ return `account-${account_id}`; } +export function streamSubject({ + project_id, + account_id, +}: { + project_id?: string; + account_id?: string; +}) { + if (project_id) { + if (account_id) { + throw Error("both account_id and project_id can't be set"); + } + return `project.${project_id}.stream.>`; + } + if (!account_id) { + throw Error("at least one of account_id and project_id must be set"); + } + return `account.${account_id}.stream.>`; +} + export function projectSubject({ project_id, compute_server_id = 0, diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index ed9502feb4..c42e0cc2f6 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -163,7 +163,6 @@ export class KV extends EventEmitter { include: "updates", key: this.filter, }); - //for await (const { key, value } of this.watch) { for await (const x of this.watch) { const { revision, key, value, sm } = x; if (this.revisions == null || this.all == null || this.times == null) { diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index e3e7ac7150..47f7649001 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -1,5 +1,5 @@ /* -Always Consistent Centralized Stream +Consistent Centralized Event Stream TODO: - ability to easily initialize with only the @@ -12,34 +12,48 @@ DEVELOPMENT: ~/cocalc/src/packages/server$ n Welcome to Node.js v18.17.1. Type ".help" for more information. -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = new a.Stream({name:'test',env,subjects:'foo.>'}); await s.init(); +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = new a.Stream({name:'test',env,subjects:'foo',filter:'foo'}); await s.init(); With browser client using a project: # in browser -> s = await cc.client.nats_client.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094', filter:'foo'}) +> s = await cc.client.nats_client.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo'}) # in node: -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = new a.Stream({name:'project-56eb622f-d398-489a-83ef-c09f1a1e8094',env,filter:'foo',subjects:'project-56eb622f-d398-489a-83ef-c09f1a1e8094.>'}); await s.init(); +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo', env}) + */ import { EventEmitter } from "events"; import { type NatsEnv } from "@cocalc/nats/types"; import { jetstreamManager, jetstream } from "@nats-io/jetstream"; -import { matchesPattern } from "@cocalc/nats/util"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { jsName, streamSubject } from "@cocalc/nats/names"; +import { nanos } from "@cocalc/nats/util"; +import { delay } from "awaiting"; + +// confirm that ephemeral consumer still exists every 15 seconds: +// In case of a long disconnect from the network, this is what +// ensures we successfully get properly updated. +const CONSUMER_MONITOR_INTERVAL = 15 * 1000; + +// Have server keep ephemeral consumers alive for an hour. This +// means even if we drop from the internet for up to an hour, the server +// doesn't forget about our consumer. But even if we are forgotten, +// the CONSUMER_MONITOR_INTERVAL ensures the event stream correctly works! +const EPHEMERAL_CONSUMER_THRESH = 60 * 60 * 1000; export class Stream extends EventEmitter { public readonly name: string; private options?; private subjects: string | string[]; private filter?: string; + private subject?: string; private env: NatsEnv; private js; private stream?; - private consumer?; private watch?; private raw: any[] = []; public readonly events: any[] = []; @@ -47,12 +61,15 @@ export class Stream extends EventEmitter { constructor({ name, env, + subject, subjects, filter, options, }: { name: string; + // subject = default subject used for publishing; defaults to filter if filter doesn't have any wildcard subjects: string | string[]; + subject?: string; filter?: string; env: NatsEnv; options?; @@ -63,6 +80,15 @@ export class Stream extends EventEmitter { this.js = jetstream(env.nc); this.name = name; this.options = options; + if ( + subject == null && + filter != null && + !filter.includes("*") && + !filter.includes(">") + ) { + subject = filter; + } + this.subject = subject; this.subjects = typeof subjects == "string" ? [subjects] : subjects; if (this.subjects.length == 0) { throw Error("subjects must be at least one string"); @@ -89,59 +115,113 @@ export class Stream extends EventEmitter { // probably already exists, so try to modify to have the requested properties. this.stream = await jsm.streams.update(this.name, options); } - this.consumer = await this.getConsumer(); this.startFetch(); }); publish = async (event, subject?) => { - if (subject != null) { - for (const pattern of this.subjects) { - if (!matchesPattern({ pattern, subject })) { - throw Error( - `subject must match subjects=${JSON.stringify(this.subjects)}`, - ); - } - } - } - subject = subject ?? this.subjects[0]; - if (this.filter) { - if (!matchesPattern({ pattern: this.filter, subject })) { - throw Error(`subject must match filter="${this.filter}"`); - } - } - return await this.js.publish(subject, this.env.jc.encode(event)); + return await this.js.publish( + subject ?? this.subject, + this.env.jc.encode(event), + ); }; - private getConsumer = async () => { + private getConsumer = async ({ startSeq }: { startSeq?: number } = {}) => { const js = jetstream(this.env.nc); const jsm = await jetstreamManager(this.env.nc); - // making an ephemeral consumer - const { name } = await jsm.consumers.add(this.name, { + // making an ephemeral consumer, which is automatically destroyed by NATS + // after inactive_threshold. At that point we MUST reset state. + const options = { filter_subject: this.filter, + inactive_threshold: nanos(EPHEMERAL_CONSUMER_THRESH), + }; + let startOptions; + if (startSeq != null) { + startOptions = { + deliver_policy: "by_start_sequence", + opt_start_seq: startSeq, + }; + } else { + startOptions = {}; + } + const { name } = await jsm.consumers.add(this.name, { + ...options, + ...startOptions, }); return await js.consumers.get(this.name, name); }; - private startFetch = async () => { - if (this.consumer == null) { - throw Error("consumer not defined"); + private startFetch = async (options?) => { + const consumer = await this.getConsumer(options); + // This goes in two stages: + // STAGE 1: Get what is in the stream now. + // First we get info so we know how many messages + // are already in the stream: + const info = await consumer.info(); + const fetch = await consumer.fetch(); + this.watch = fetch; + let i = 0; + // grab the messages. This should be very efficient since it + // internally grabs them in batches. + for await (const mesg of fetch) { + this.handle(mesg, true); + i += 1; + if (i >= info.num_pending) { + break; + } } - const consumer = this.consumer; - this.watch = await consumer.fetch(); - for await (const mesg of this.watch) { - this.handle(mesg); + if (this.stream == null) { + // closed *during* initial load + return; + } + + this.monitorConsumer(consumer); + + // STAGE 2: Watch for new events. It's the same consumer though, + // so we are **guaranteed** not to miss anything. + this.emit("connected"); + const consume = await consumer.consume(); + this.watch = consume; + for await (const mesg of consume) { + this.handle(mesg, false); } }; - private handle = (mesg) => { - let data; + private monitorConsumer = async (consumer) => { + while (this.stream != null) { + try { + await consumer.info(); + } catch (err) { + // console.log(`monitorConsumer -- got err ${err}`); + if ( + err.name == "ConsumerNotFoundError" || + err.code == 10014 || + err.message == "consumer not found" + ) { + // if it is a consumer not found error, we make a new consumer, + // starting AFTER the last event we retrieved + this.watch.stop(); // stop current watch + // make new one: + const startSeq = this.raw[this.raw.length - 1]?.seq + 1; + this.startFetch({ startSeq }); + return; // because startFetch creates a new consumer monitor loop + } + } + } + await delay(CONSUMER_MONITOR_INTERVAL); + }; + + private handle = (raw, noEmit = false) => { + let event; try { - data = this.env.jc.decode(mesg.data); + event = this.env.jc.decode(raw.data); } catch { - data = mesg.data; + event = raw.data; + } + this.events.push(event); + this.raw.push(raw); + if (!noEmit) { + this.emit("change", event, raw); } - this.events.push(data); - this.raw.push(mesg); }; close = () => { @@ -151,8 +231,49 @@ export class Stream extends EventEmitter { this.watch.stop(); delete this.watch; delete this.stream; - delete this.consumer; this.emit("closed"); this.removeAllListeners(); }; } + +// One stream for each account and one for each project. +// Use the filters to restrict, e.g., to events about a particular file. + +const streamCache: { [key: string]: Stream } = {}; +export const stream = reuseInFlight( + async ({ + env, + account_id, + project_id, + name, + }: { + env: NatsEnv; + name: string; + account_id?: string; + project_id?: string; + }) => { + const jsname = jsName({ account_id, project_id }); + const subjects = streamSubject({ account_id, project_id }); + const filter = subjects.replace(">", name); + const key = JSON.stringify([name, jsname]); + if (streamCache[key] == null) { + const stream = new Stream({ + name: jsname, + subjects, + subject: filter, + filter, + env, + }); + await stream.init(); + streamCache[key] = stream; + stream.on("closed", () => { + delete streamCache[key]; + }); + } + return streamCache[key]; + }, + { + createKey: (args) => + JSON.stringify([args[0].account_id, args[0].project_id, args[0].name]), + }, +); diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index 3369112620..b15e05a75e 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -98,3 +98,13 @@ export function matchesPattern({ return i === subParts.length && j === patParts.length; } + +// Converts the specified millis into Nanos +export function nanos(millis: number): number { + return millis * 1000000; +} + +// Convert the specified Nanos into millis +export function millis(ns: number): number { + return Math.floor(ns / 1000000); +} diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index fa7b18d720..4f9c99e517 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -18,7 +18,7 @@ DOCS: USAGE: -a = require('@cocalc/server/nats/auth'); await a.configureNatsUser({account_id:'275f1db7-bf37-4b44-b9aa-d64694269c9f'}) +a = require('@cocalc/server/nats/auth'); await a.configureNatsUser({account_id:'6aae57c6-08f1-4bb5-848b-3ceb53e61ede'}) await a.configureNatsUser({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}) */ From d4deba6e220e7f43031a86be6f1f4d5a9d231ea9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 8 Feb 2025 02:18:14 +0000 Subject: [PATCH 137/281] nats: create dstream = distributed eventually consistent stream class --- src/packages/frontend/nats/client.ts | 9 ++ src/packages/nats/sync/dkv.ts | 3 + src/packages/nats/sync/dstream.ts | 140 +++++++++++++++++++++++++++ src/packages/nats/sync/stream.ts | 55 ++++++----- src/packages/util/types/index.ts | 10 +- 5 files changed, 190 insertions(+), 27 deletions(-) create mode 100644 src/packages/nats/sync/dstream.ts diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 2afdbab45f..ad565c007e 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -22,6 +22,7 @@ import { SystemKv } from "@cocalc/nats/system"; import { KV } from "@cocalc/nats/sync/kv"; import { DKV } from "@cocalc/nats/sync/dkv"; import { stream } from "@cocalc/nats/sync/stream"; +import { dstream } from "@cocalc/nats/sync/dstream"; import { initApi } from "@cocalc/frontend/nats/api"; import { delay } from "awaiting"; import { Svcm } from "@nats-io/services"; @@ -464,6 +465,14 @@ export class NatsClient { return await stream({ ...opts, env: await this.getEnv() }); }; + dstream = async (opts: { + account_id?: string; + project_id?: string; + name: string; + }) => { + return await dstream({ ...opts, env: await this.getEnv() }); + }; + microservicesClient = async () => { const nc = await this.getConnection(); // @ts-ignore diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index 1486a0b76e..a253f07dea 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -216,6 +216,9 @@ export class DKV extends EventEmitter { }; hasUnsavedChanges = () => { + if (this.kv == null) { + return false; + } return this.changed.size > 0 || Object.keys(this.local).length > 0; }; diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts new file mode 100644 index 0000000000..7976093205 --- /dev/null +++ b/src/packages/nats/sync/dstream.ts @@ -0,0 +1,140 @@ +/* +Eventually Consistent Distributed Event Stream + +DEVELOPMENT: + + +# in node: +> env = await require("@cocalc/backend/nats/env").getEnv() + a = require("@cocalc/nats/sync/dstream"); s = await a.dstream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo', env}) + + +*/ + +import { EventEmitter } from "events"; +import { Stream, type StreamOptions, type UserStreamOptions } from "./stream"; +import { jsName, streamSubject, randomId } from "@cocalc/nats/names"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { type JSONValue } from "@cocalc/util/types"; +import { delay } from "awaiting"; +import { map as awaitMap } from "awaiting"; +const MAX_PARALLEL = 50; + +export class DStream extends EventEmitter { + private stream?: Stream; + private events: JSONValue[]; + private raw: any[]; + private local: { [id: string]: { event: JSONValue; subject?: string } } = {}; + + constructor(opts: StreamOptions) { + super(); + this.stream = new Stream(opts); + this.events = this.stream.events; + this.raw = this.stream.raw; + } + + init = reuseInFlight(async () => { + if (this.stream == null) { + throw Error("closed"); + } + this.stream.on("change", (...args) => { + this.emit("change", ...args); + }); + await this.stream.init(); + this.emit("connected"); + }); + + close = () => { + if (this.stream == null) { + return; + } + this.stream.close(); + this.emit("closed"); + this.removeAllListeners(); + delete this.stream; + // @ts-ignore + delete this.local; + // @ts-ignore + delete this.events; + // @ts-ignore + delete this.raw; + }; + + publish = (event: JSONValue, subject?: string) => { + const id = randomId(); + this.local[id] = { event, subject }; + this.save(); + }; + + hasUnsavedChanges = () => { + if (this.stream == null) { + return false; + } + return Object.keys(this.local).length > 0; + }; + + unsavedChanges = () => { + return Object.values(this.local); + }; + + private save = reuseInFlight(async () => { + let d = 100; + while (true) { + try { + await this.attemptToSave(); + //console.log("successfully saved"); + } catch { + //(err) { + // console.log("problem saving", err); + } + if (this.hasUnsavedChanges()) { + d = Math.min(10000, d * 1.3) + Math.random() * 100; + await delay(d); + } else { + return; + } + } + }); + + private attemptToSave = reuseInFlight(async () => { + const f = async (id) => { + if (this.stream == null) { + throw Error("closed"); + } + const { event, subject } = this.local[id]; + // @ts-ignore + await this.stream.publish(event, subject, { msgID: id }); + delete this.local[id]; + }; + await awaitMap(Object.keys(this.local), MAX_PARALLEL, f); + }); +} + +const dstreamCache: { [key: string]: DStream } = {}; +export const dstream = reuseInFlight( + async ({ env, account_id, project_id, name }: UserStreamOptions) => { + const jsname = jsName({ account_id, project_id }); + const subjects = streamSubject({ account_id, project_id }); + const filter = subjects.replace(">", name); + const key = JSON.stringify([name, jsname]); + if (dstreamCache[key] == null) { + const dstream = new DStream({ + name: jsname, + subjects, + subject: filter, + filter, + env, + }); + await dstream.init(); + dstreamCache[key] = dstream; + dstream.on("closed", () => { + delete dstreamCache[key]; + }); + } + return dstreamCache[key]; + }, + { + createKey: (args) => + JSON.stringify([args[0].account_id, args[0].project_id, args[0].name]), + }, +); diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 47f7649001..7814e18d7c 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -2,10 +2,10 @@ Consistent Centralized Event Stream TODO: - - ability to easily initialize with only the - last n messages or only messages going back - to time t - + - ability to easily initialize with only the last n messages or + only messages going back to time t + - automatically delete data according to various rules, e.g., needed + for terminal, but DEVELOPMENT: @@ -45,6 +45,16 @@ const CONSUMER_MONITOR_INTERVAL = 15 * 1000; // the CONSUMER_MONITOR_INTERVAL ensures the event stream correctly works! const EPHEMERAL_CONSUMER_THRESH = 60 * 60 * 1000; +export interface StreamOptions { + name: string; + // subject = default subject used for publishing; defaults to filter if filter doesn't have any wildcard + subjects: string | string[]; + subject?: string; + filter?: string; + env: NatsEnv; + options?; +} + export class Stream extends EventEmitter { public readonly name: string; private options?; @@ -55,7 +65,8 @@ export class Stream extends EventEmitter { private js; private stream?; private watch?; - private raw: any[] = []; + // don't do "this.raw=" or "this.events=" anywhere in this class! + public readonly raw: any[] = []; public readonly events: any[] = []; constructor({ @@ -65,15 +76,7 @@ export class Stream extends EventEmitter { subjects, filter, options, - }: { - name: string; - // subject = default subject used for publishing; defaults to filter if filter doesn't have any wildcard - subjects: string | string[]; - subject?: string; - filter?: string; - env: NatsEnv; - options?; - }) { + }: StreamOptions) { super(); this.env = env; // create a jetstream client so we can publish to the stream @@ -104,6 +107,8 @@ export class Stream extends EventEmitter { const options = { subjects: this.subjects, compression: "s2", + // our streams are relatively small so a longer duplicate window than 2 minutes seems ok. + duplicate_window: nanos(1000 * 60 * 15), ...this.options, }; try { @@ -118,10 +123,11 @@ export class Stream extends EventEmitter { this.startFetch(); }); - publish = async (event, subject?) => { + publish = async (event: any, subject?: string, options?) => { return await this.js.publish( subject ?? this.subject, this.env.jc.encode(event), + options, ); }; @@ -239,19 +245,16 @@ export class Stream extends EventEmitter { // One stream for each account and one for each project. // Use the filters to restrict, e.g., to events about a particular file. +export interface UserStreamOptions { + env: NatsEnv; + name: string; + account_id?: string; + project_id?: string; +} + const streamCache: { [key: string]: Stream } = {}; export const stream = reuseInFlight( - async ({ - env, - account_id, - project_id, - name, - }: { - env: NatsEnv; - name: string; - account_id?: string; - project_id?: string; - }) => { + async ({ env, account_id, project_id, name }: UserStreamOptions) => { const jsname = jsName({ account_id, project_id }); const subjects = streamSubject({ account_id, project_id }); const filter = subjects.replace(">", name); diff --git a/src/packages/util/types/index.ts b/src/packages/util/types/index.ts index d766c91c44..ff9c022d7f 100644 --- a/src/packages/util/types/index.ts +++ b/src/packages/util/types/index.ts @@ -1,6 +1,14 @@ /* Misc types that are used in frontends, backends, etc. -*/ + */ export type { DirectoryListingEntry } from "./directory-listing"; export type { DatastoreConfig } from "./datastore"; + +export type JSONValue = + | string + | number + | boolean + | null + | { [key: string]: JSONValue } + | JSONValue[]; From 6ff903bf138b8a62770e70beebe9d0b0b011a272 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 8 Feb 2025 03:50:19 +0000 Subject: [PATCH 138/281] nats: add stream limits --- src/packages/nats/sync/dstream.ts | 34 +++++++---- src/packages/nats/sync/stream.ts | 98 +++++++++++++++++++++++++------ 2 files changed, 103 insertions(+), 29 deletions(-) diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 7976093205..7602acbc24 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -15,21 +15,20 @@ import { EventEmitter } from "events"; import { Stream, type StreamOptions, type UserStreamOptions } from "./stream"; import { jsName, streamSubject, randomId } from "@cocalc/nats/names"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { type JSONValue } from "@cocalc/util/types"; import { delay } from "awaiting"; import { map as awaitMap } from "awaiting"; const MAX_PARALLEL = 50; export class DStream extends EventEmitter { private stream?: Stream; - private events: JSONValue[]; + private messages: any[]; private raw: any[]; - private local: { [id: string]: { event: JSONValue; subject?: string } } = {}; + private local: { [id: string]: { mesg: any; subject?: string } } = {}; constructor(opts: StreamOptions) { super(); this.stream = new Stream(opts); - this.events = this.stream.events; + this.messages = this.stream.messages; this.raw = this.stream.raw; } @@ -55,14 +54,18 @@ export class DStream extends EventEmitter { // @ts-ignore delete this.local; // @ts-ignore - delete this.events; + delete this.messages; // @ts-ignore delete this.raw; }; - publish = (event: JSONValue, subject?: string) => { + get = () => { + return [...this.messages, ...Object.values(this.local)]; + }; + + publish = (mesg, subject?: string) => { const id = randomId(); - this.local[id] = { event, subject }; + this.local[id] = { mesg, subject }; this.save(); }; @@ -101,28 +104,30 @@ export class DStream extends EventEmitter { if (this.stream == null) { throw Error("closed"); } - const { event, subject } = this.local[id]; + const { mesg, subject } = this.local[id]; // @ts-ignore - await this.stream.publish(event, subject, { msgID: id }); + await this.stream.publish(mesg, subject, { msgID: id }); delete this.local[id]; }; + // NOTE: ES6 spec guarantees "String keys are returned in the order in which they were added to the object." await awaitMap(Object.keys(this.local), MAX_PARALLEL, f); }); } const dstreamCache: { [key: string]: DStream } = {}; export const dstream = reuseInFlight( - async ({ env, account_id, project_id, name }: UserStreamOptions) => { + async ({ env, account_id, project_id, name, limits }: UserStreamOptions) => { const jsname = jsName({ account_id, project_id }); const subjects = streamSubject({ account_id, project_id }); const filter = subjects.replace(">", name); - const key = JSON.stringify([name, jsname]); + const key = JSON.stringify([name, jsname, limits]); if (dstreamCache[key] == null) { const dstream = new DStream({ name: jsname, subjects, subject: filter, filter, + limits, env, }); await dstream.init(); @@ -135,6 +140,11 @@ export const dstream = reuseInFlight( }, { createKey: (args) => - JSON.stringify([args[0].account_id, args[0].project_id, args[0].name]), + JSON.stringify([ + args[0].account_id, + args[0].project_id, + args[0].name, + args[0].limits, + ]), }, ); diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 7814e18d7c..d6b84aa9e1 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -1,5 +1,5 @@ /* -Consistent Centralized Event Stream +Consistent Centralized Event Stream = ordered list of messages TODO: - ability to easily initialize with only the last n messages or @@ -24,6 +24,10 @@ With browser client using a project: > env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo', env}) +# Involving a limit: + +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo', env, limits:{maxMessages:20}}) + */ import { EventEmitter } from "events"; @@ -33,6 +37,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { jsName, streamSubject } from "@cocalc/nats/names"; import { nanos } from "@cocalc/nats/util"; import { delay } from "awaiting"; +import { debounce } from "lodash"; // confirm that ephemeral consumer still exists every 15 seconds: // In case of a long disconnect from the network, this is what @@ -45,6 +50,14 @@ const CONSUMER_MONITOR_INTERVAL = 15 * 1000; // the CONSUMER_MONITOR_INTERVAL ensures the event stream correctly works! const EPHEMERAL_CONSUMER_THRESH = 60 * 60 * 1000; +interface StreamLimitOptions { + maxMessages?: number; + maxSize?: { + maxTotalSize: number; + getMessageSize: (message) => number; + }; +} + export interface StreamOptions { name: string; // subject = default subject used for publishing; defaults to filter if filter doesn't have any wildcard @@ -52,22 +65,25 @@ export interface StreamOptions { subject?: string; filter?: string; env: NatsEnv; - options?; + natsStreamOptions?; + limits?: StreamLimitOptions; } export class Stream extends EventEmitter { public readonly name: string; - private options?; + private natsStreamOptions?; + private limits?: StreamLimitOptions; private subjects: string | string[]; private filter?: string; private subject?: string; private env: NatsEnv; private js; + private jsm; private stream?; private watch?; - // don't do "this.raw=" or "this.events=" anywhere in this class! + // don't do "this.raw=" or "this.messages=" anywhere in this class! public readonly raw: any[] = []; - public readonly events: any[] = []; + public readonly messages: any[] = []; constructor({ name, @@ -75,14 +91,15 @@ export class Stream extends EventEmitter { subject, subjects, filter, - options, + natsStreamOptions, + limits, }: StreamOptions) { super(); this.env = env; // create a jetstream client so we can publish to the stream this.js = jetstream(env.nc); this.name = name; - this.options = options; + this.natsStreamOptions = natsStreamOptions; if ( subject == null && filter != null && @@ -97,33 +114,39 @@ export class Stream extends EventEmitter { throw Error("subjects must be at least one string"); } this.filter = filter; + this.limits = limits; } init = reuseInFlight(async () => { if (this.stream != null) { return; } - const jsm = await jetstreamManager(this.env.nc); + this.jsm = await jetstreamManager(this.env.nc); const options = { subjects: this.subjects, compression: "s2", // our streams are relatively small so a longer duplicate window than 2 minutes seems ok. duplicate_window: nanos(1000 * 60 * 15), - ...this.options, + ...this.natsStreamOptions, }; try { - this.stream = await jsm.streams.add({ + this.stream = await this.jsm.streams.add({ name: this.name, ...options, }); } catch (err) { // probably already exists, so try to modify to have the requested properties. - this.stream = await jsm.streams.update(this.name, options); + this.stream = await this.jsm.streams.update(this.name, options); } this.startFetch(); }); + get = () => { + return [...this.messages]; + }; + publish = async (event: any, subject?: string, options?) => { + this.expire(); return await this.js.publish( subject ?? this.subject, this.env.jc.encode(event), @@ -182,7 +205,7 @@ export class Stream extends EventEmitter { this.monitorConsumer(consumer); - // STAGE 2: Watch for new events. It's the same consumer though, + // STAGE 2: Watch for new mesg. It's the same consumer though, // so we are **guaranteed** not to miss anything. this.emit("connected"); const consume = await consumer.consume(); @@ -223,11 +246,12 @@ export class Stream extends EventEmitter { } catch { event = raw.data; } - this.events.push(event); + this.messages.push(event); this.raw.push(raw); if (!noEmit) { this.emit("change", event, raw); } + this.expire(); }; close = () => { @@ -237,34 +261,69 @@ export class Stream extends EventEmitter { this.watch.stop(); delete this.watch; delete this.stream; + delete this.jsm; this.emit("closed"); this.removeAllListeners(); }; + + // ensure any limits are satisfied, i.e., delete old messages. + private expire = debounce( + reuseInFlight(async () => { + if (this.limits == null || this.jsm == null) { + return; + } + const maxMessages = this.limits.maxMessages ?? 0; + if (maxMessages > 0 && this.messages.length > maxMessages) { + // ensure there are at most this.limits.maxMessages messages + // by deleting this oldest ones + const i = this.messages.length - maxMessages + 1; + if (i >= 0 && i < this.messages.length) { + const { seq } = this.raw[i]; + try { + await this.jsm.streams.purge(this.name, { + filter: this.filter, + seq, + }); + this.messages.splice(0, i - 1); + this.raw.splice(0, i - 1); + } catch (err) { + if (err.code != "TIMEOUT") { + console.log(`WARNING: discarding old messages - ${err}`); + } + } + } + } + }), + 3000, + { leading: false, trailing: true }, + ); } // One stream for each account and one for each project. -// Use the filters to restrict, e.g., to events about a particular file. +// Use the filters to restrict, e.g., to message about a particular file. export interface UserStreamOptions { env: NatsEnv; name: string; account_id?: string; project_id?: string; + limits?: StreamLimitOptions; } const streamCache: { [key: string]: Stream } = {}; export const stream = reuseInFlight( - async ({ env, account_id, project_id, name }: UserStreamOptions) => { + async ({ env, account_id, project_id, name, limits }: UserStreamOptions) => { const jsname = jsName({ account_id, project_id }); const subjects = streamSubject({ account_id, project_id }); const filter = subjects.replace(">", name); - const key = JSON.stringify([name, jsname]); + const key = JSON.stringify([name, jsname, limits]); if (streamCache[key] == null) { const stream = new Stream({ name: jsname, subjects, subject: filter, filter, + limits, env, }); await stream.init(); @@ -277,6 +336,11 @@ export const stream = reuseInFlight( }, { createKey: (args) => - JSON.stringify([args[0].account_id, args[0].project_id, args[0].name]), + JSON.stringify([ + args[0].account_id, + args[0].project_id, + args[0].name, + args[0].limits, + ]), }, ); From 98b6ac284890c179e730e923116fcabf98145475 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 8 Feb 2025 15:34:18 +0000 Subject: [PATCH 139/281] nats filtered stream: implement all the limits --- src/packages/nats/sync/stream.ts | 202 ++++++++++++++++++++++++------- src/packages/nats/util.ts | 6 +- 2 files changed, 159 insertions(+), 49 deletions(-) diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index d6b84aa9e1..23ccd363d3 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -2,10 +2,11 @@ Consistent Centralized Event Stream = ordered list of messages TODO: - - ability to easily initialize with only the last n messages or - only messages going back to time t - - automatically delete data according to various rules, e.g., needed - for terminal, but + - ability to easily initialize with only the most recent n messages or + only messages starting at time t. + - maybe the limits and other config should be stored in a KV store so + they are sync'd between clients automatically. That's what NATS surely + does internally. DEVELOPMENT: @@ -24,10 +25,13 @@ With browser client using a project: > env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo', env}) -# Involving a limit: +# Involving limits: -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo', env, limits:{maxMessages:20}}) +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo', env, limits:{max_msgs:5,max_age:1000000*1000*15,max_bytes:10000,max_msg_size:1000}}) +> s.get() +In browser: +> s = await cc.client.nats_client.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo',limits:{max_msgs:5,max_age:1000000*1000*15,max_bytes:10000,max_msg_size:1000}}) */ import { EventEmitter } from "events"; @@ -35,9 +39,9 @@ import { type NatsEnv } from "@cocalc/nats/types"; import { jetstreamManager, jetstream } from "@nats-io/jetstream"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { jsName, streamSubject } from "@cocalc/nats/names"; -import { nanos } from "@cocalc/nats/util"; +import { nanos, type Nanos } from "@cocalc/nats/util"; import { delay } from "awaiting"; -import { debounce } from "lodash"; +import { throttle } from "lodash"; // confirm that ephemeral consumer still exists every 15 seconds: // In case of a long disconnect from the network, this is what @@ -50,12 +54,29 @@ const CONSUMER_MONITOR_INTERVAL = 15 * 1000; // the CONSUMER_MONITOR_INTERVAL ensures the event stream correctly works! const EPHEMERAL_CONSUMER_THRESH = 60 * 60 * 1000; -interface StreamLimitOptions { - maxMessages?: number; - maxSize?: { - maxTotalSize: number; - getMessageSize: (message) => number; - }; +// We re-implement exactly the same stream-wide limits that NATS has, +// but instead, these are for the stream **with the given filter**. +// Limits are enforced by all clients *client side* within a few seconds of any +// client making changes. +// For API consistency, max_age is is in nano-seconds. Also, obviously +// the true limit is the minimum of the full NATS stream limits and +// these limits. +const ENFORCE_LIMITS_THROTTLE_MS = 3000; +interface FilteredStreamLimitOptions { + // How many messages may be in a Stream, oldest messages will be removed + // if the Stream exceeds this size. -1 for unlimited. + max_msgs: number; + // Maximum age of any message in the stream matching the filter, + // expressed in nanoseconds. 0 for unlimited. + // Use 'import {nanos} from "@cocalc/nats/util"' then "nanos(milliseconds)" to give input in ms. + max_age: Nanos; + // How big the Stream may be, when the combined stream size matching the filter + // exceeds this old messages are removed. -1 for unlimited. + // This is enforced only on write, so if you change it, it only applies + // to future messages. + max_bytes: number; + // The largest message that will be accepted by the Stream. -1 for unlimited. + max_msg_size: number; } export interface StreamOptions { @@ -66,13 +87,13 @@ export interface StreamOptions { filter?: string; env: NatsEnv; natsStreamOptions?; - limits?: StreamLimitOptions; + limits?: Partial; } export class Stream extends EventEmitter { public readonly name: string; private natsStreamOptions?; - private limits?: StreamLimitOptions; + private limits: FilteredStreamLimitOptions; private subjects: string | string[]; private filter?: string; private subject?: string; @@ -114,7 +135,13 @@ export class Stream extends EventEmitter { throw Error("subjects must be at least one string"); } this.filter = filter; - this.limits = limits; + this.limits = { + max_msgs: -1, + max_age: 0, + max_bytes: -1, + max_msg_size: -1, + ...limits, + }; } init = reuseInFlight(async () => { @@ -145,13 +172,23 @@ export class Stream extends EventEmitter { return [...this.messages]; }; - publish = async (event: any, subject?: string, options?) => { - this.expire(); - return await this.js.publish( - subject ?? this.subject, - this.env.jc.encode(event), - options, - ); + publish = async (mesg: any, subject?: string, options?) => { + if (this.js == null) { + throw Error("closed"); + } + const data = this.env.jc.encode(mesg); + if ( + this.limits.max_msg_size > -1 && + data.length > this.limits.max_msg_size + ) { + throw Error( + `message size exceeds max_msg_size=${this.limits.max_msg_size} bytes`, + ); + } + this.enforceLimits(); + const resp = await this.js.publish(subject ?? this.subject, data, options); + this.enforceLimits(); + return resp; }; private getConsumer = async ({ startSeq }: { startSeq?: number } = {}) => { @@ -207,11 +244,13 @@ export class Stream extends EventEmitter { // STAGE 2: Watch for new mesg. It's the same consumer though, // so we are **guaranteed** not to miss anything. + this.enforceLimits(); this.emit("connected"); const consume = await consumer.consume(); this.watch = consume; for await (const mesg of consume) { this.handle(mesg, false); + this.enforceLimits(); } }; @@ -251,7 +290,6 @@ export class Stream extends EventEmitter { if (!noEmit) { this.emit("change", event, raw); } - this.expire(); }; close = () => { @@ -262,40 +300,110 @@ export class Stream extends EventEmitter { delete this.watch; delete this.stream; delete this.jsm; + delete this.js; this.emit("closed"); this.removeAllListeners(); }; + // delete all messages up to and including the + // one at position index, i.e., this.messages[index] + // is deleted. + // NOTE: other clients will NOT see the result of a purge, + // except when done implicitly via limits, since all clients + // truncate this.raw and this.messages directly. + purge = async ({ index = -1 }: { index?: number } = {}) => { + // console.log("purge", { index }); + if (index >= this.raw.length - 1 || index == -1) { + index = this.raw.length - 1; + // everything + // console.log("purge everything"); + await this.jsm.streams.purge(this.name, { + filter: this.filter, + }); + } else { + const { seq } = this.raw[index + 1]; + await this.jsm.streams.purge(this.name, { + filter: this.filter, + seq, + }); + } + this.messages.splice(0, index + 1); + this.raw.splice(0, index + 1); + }; + // ensure any limits are satisfied, i.e., delete old messages. - private expire = debounce( + private enforceLimits = throttle( reuseInFlight(async () => { - if (this.limits == null || this.jsm == null) { + if (this.jsm == null) { return; } - const maxMessages = this.limits.maxMessages ?? 0; - if (maxMessages > 0 && this.messages.length > maxMessages) { - // ensure there are at most this.limits.maxMessages messages - // by deleting this oldest ones - const i = this.messages.length - maxMessages + 1; - if (i >= 0 && i < this.messages.length) { - const { seq } = this.raw[i]; - try { - await this.jsm.streams.purge(this.name, { - filter: this.filter, - seq, - }); - this.messages.splice(0, i - 1); - this.raw.splice(0, i - 1); - } catch (err) { - if (err.code != "TIMEOUT") { - console.log(`WARNING: discarding old messages - ${err}`); + const { max_msgs, max_age, max_bytes } = this.limits; + // we check with each defined limit if some old messages + // should be dropped, and if so move limit forward. If + // it is above -1 at the end, we do the drop. + let index = -1; + const setIndex = (i, _limit) => { + // console.log("setIndex", { i, _limit }); + index = Math.max(i, index); + }; + //max_msgs + if (max_msgs > -1 && this.messages.length > max_msgs) { + // ensure there are at most this.limits.max_msgs messages + // by deleting the oldest ones up to a specified point. + const i = this.messages.length - max_msgs; + if (i > 0) { + setIndex(i - 1, "max_msgs"); + } + } + + // max_age + if (max_age > 0) { + // expire messages older than max_age nanoseconds + const recent = this.raw[this.raw.length - 1]; + if (recent != null) { + // to avoid potential clock skew, we define *now* as the time of the most + // recent message. For us, this should be fine, since we only impose limits + // when writing new messages, and none of these limits are guaranteed. + const now = recent.info.timestampNanos; + const cutoff = now - max_age; + for (let i = this.raw.length - 1; i >= 0; i--) { + if (this.raw[i].info.timestampNanos < cutoff) { + // it just went over the limit. Everything before + // and including the i-th message must be deleted. + setIndex(i, "max_age"); + break; } } } } + + // max_bytes + if (max_bytes >= 0) { + let t = 0; + for (let i = this.raw.length - 1; i >= 0; i--) { + t += this.raw[i].data.length; + if (t > max_bytes) { + // it just went over the limit. Everything before + // and including the i-th message must be deleted. + setIndex(i, "max_bytes"); + break; + } + } + } + + if (index > -1) { + try { + // console.log("imposing limit via purge ", { index }); + await this.purge({ index }); + } catch (err) { + if (err.code != "TIMEOUT") { + console.log(`WARNING: purging old messages - ${err}`); + } + } + } }), - 3000, - { leading: false, trailing: true }, + ENFORCE_LIMITS_THROTTLE_MS, + { leading: true, trailing: true }, ); } @@ -307,7 +415,7 @@ export interface UserStreamOptions { name: string; account_id?: string; project_id?: string; - limits?: StreamLimitOptions; + limits?: FilteredStreamLimitOptions; } const streamCache: { [key: string]: Stream } = {}; diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index b15e05a75e..fa010a7f68 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -100,11 +100,13 @@ export function matchesPattern({ } // Converts the specified millis into Nanos -export function nanos(millis: number): number { +export type Nanos = number; +export function nanos(millis: number): Nanos { return millis * 1000000; } // Convert the specified Nanos into millis -export function millis(ns: number): number { +export function millis(ns: Nanos): number { return Math.floor(ns / 1000000); } + From 3c461b996604a58f2e6564feab4b0c2ab56b9569 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 8 Feb 2025 16:57:24 +0000 Subject: [PATCH 140/281] nats streams: support start_seq --- src/packages/nats/sync/stream.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 23ccd363d3..548f4d927c 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -2,8 +2,8 @@ Consistent Centralized Event Stream = ordered list of messages TODO: - - ability to easily initialize with only the most recent n messages or - only messages starting at time t. + - ability to easily initialize with only messages starting at a given seq + - load old messages starting at a given seq. - maybe the limits and other config should be stored in a KV store so they are sync'd between clients automatically. That's what NATS surely does internally. @@ -32,6 +32,8 @@ With browser client using a project: In browser: > s = await cc.client.nats_client.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo',limits:{max_msgs:5,max_age:1000000*1000*15,max_bytes:10000,max_msg_size:1000}}) + + */ import { EventEmitter } from "events"; @@ -88,6 +90,8 @@ export interface StreamOptions { env: NatsEnv; natsStreamOptions?; limits?: Partial; + // only load historic messages starting at the given seq number. + start_seq?: number; } export class Stream extends EventEmitter { @@ -98,6 +102,7 @@ export class Stream extends EventEmitter { private filter?: string; private subject?: string; private env: NatsEnv; + private start_seq?: number; private js; private jsm; private stream?; @@ -114,6 +119,7 @@ export class Stream extends EventEmitter { filter, natsStreamOptions, limits, + start_seq, }: StreamOptions) { super(); this.env = env; @@ -135,6 +141,7 @@ export class Stream extends EventEmitter { throw Error("subjects must be at least one string"); } this.filter = filter; + this.start_seq = start_seq; this.limits = { max_msgs: -1, max_age: 0, @@ -201,6 +208,9 @@ export class Stream extends EventEmitter { inactive_threshold: nanos(EPHEMERAL_CONSUMER_THRESH), }; let startOptions; + if (startSeq == null && this.start_seq != null) { + startSeq = this.start_seq; + } if (startSeq != null) { startOptions = { deliver_policy: "by_start_sequence", From 18b84331ac661e3e95ddb018b4a417a7026a9f92 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 8 Feb 2025 17:08:36 +0000 Subject: [PATCH 141/281] nats streams: better cache --- src/packages/nats/sync/dstream.ts | 23 +++++++++++------------ src/packages/nats/sync/stream.ts | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 7602acbc24..2c3b2ecedc 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -12,7 +12,12 @@ DEVELOPMENT: */ import { EventEmitter } from "events"; -import { Stream, type StreamOptions, type UserStreamOptions } from "./stream"; +import { + Stream, + type StreamOptions, + type UserStreamOptions, + userStreamOptionsKey, +} from "./stream"; import { jsName, streamSubject, randomId } from "@cocalc/nats/names"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { delay } from "awaiting"; @@ -116,19 +121,19 @@ export class DStream extends EventEmitter { const dstreamCache: { [key: string]: DStream } = {}; export const dstream = reuseInFlight( - async ({ env, account_id, project_id, name, limits }: UserStreamOptions) => { + async (options: UserStreamOptions) => { + const { account_id, project_id, name } = options; const jsname = jsName({ account_id, project_id }); const subjects = streamSubject({ account_id, project_id }); const filter = subjects.replace(">", name); - const key = JSON.stringify([name, jsname, limits]); + const key = userStreamOptionsKey(options); if (dstreamCache[key] == null) { const dstream = new DStream({ + ...options, name: jsname, subjects, subject: filter, filter, - limits, - env, }); await dstream.init(); dstreamCache[key] = dstream; @@ -139,12 +144,6 @@ export const dstream = reuseInFlight( return dstreamCache[key]; }, { - createKey: (args) => - JSON.stringify([ - args[0].account_id, - args[0].project_id, - args[0].name, - args[0].limits, - ]), + createKey: (args) => userStreamOptionsKey(args[0]), }, ); diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 548f4d927c..e5068404ea 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -426,23 +426,29 @@ export interface UserStreamOptions { account_id?: string; project_id?: string; limits?: FilteredStreamLimitOptions; + start_seq?: number; +} + +export function userStreamOptionsKey(options: UserStreamOptions) { + const { env, ...x } = options; + return JSON.stringify(x); } const streamCache: { [key: string]: Stream } = {}; export const stream = reuseInFlight( - async ({ env, account_id, project_id, name, limits }: UserStreamOptions) => { + async (options: UserStreamOptions) => { + const { account_id, project_id, name } = options; const jsname = jsName({ account_id, project_id }); const subjects = streamSubject({ account_id, project_id }); const filter = subjects.replace(">", name); - const key = JSON.stringify([name, jsname, limits]); + const key = userStreamOptionsKey(options); if (streamCache[key] == null) { const stream = new Stream({ + ...options, name: jsname, subjects, subject: filter, filter, - limits, - env, }); await stream.init(); streamCache[key] = stream; @@ -453,12 +459,6 @@ export const stream = reuseInFlight( return streamCache[key]; }, { - createKey: (args) => - JSON.stringify([ - args[0].account_id, - args[0].project_id, - args[0].name, - args[0].limits, - ]), + createKey: (args) => userStreamOptionsKey(args[0]), }, ); From bae0e05da54513602baa34ac3a5d6cfdf503967f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 8 Feb 2025 17:22:40 +0000 Subject: [PATCH 142/281] nats: easy use of account/project streams from backend --- src/packages/backend/nats/sync.ts | 11 +++++++++++ src/packages/backend/package.json | 1 + src/packages/backend/tsconfig.json | 2 +- src/packages/nats/sync/dstream.ts | 9 ++++++++- src/packages/nats/sync/stream.ts | 10 +++++++++- src/packages/pnpm-lock.yaml | 3 +++ 6 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 src/packages/backend/nats/sync.ts diff --git a/src/packages/backend/nats/sync.ts b/src/packages/backend/nats/sync.ts new file mode 100644 index 0000000000..5f868ab5bd --- /dev/null +++ b/src/packages/backend/nats/sync.ts @@ -0,0 +1,11 @@ +import { stream as createStream } from "@cocalc/nats/sync/stream"; +import { dstream as createDstream } from "@cocalc/nats/sync/dstream"; +import { getEnv } from "@cocalc/backend/nats/env"; + +export async function stream(opts) { + return await createStream({ ...opts, env: await getEnv() }); +} + +export async function dstream(opts) { + return await createDstream({ ...opts, env: await getEnv() }); +} diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 67ae23cfd2..ddb8a4dde9 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -24,6 +24,7 @@ "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", + "@cocalc/nats": "workspace:*", "@cocalc/util": "workspace:*", "@types/debug": "^4.1.12", "@types/watchpack": "^2.4.4", diff --git a/src/packages/backend/tsconfig.json b/src/packages/backend/tsconfig.json index 43ebbc564b..8d855cf7c6 100644 --- a/src/packages/backend/tsconfig.json +++ b/src/packages/backend/tsconfig.json @@ -7,5 +7,5 @@ "outDir": "dist" }, "exclude": ["node_modules", "dist", "test"], - "references": [{ "path": "../util" }] + "references": [{ "path": "../util", "path": "../nats" }] } diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 2c3b2ecedc..f5250250a2 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -4,7 +4,10 @@ Eventually Consistent Distributed Event Stream DEVELOPMENT: -# in node: +# in node -- note the package directory!! +~/cocalc/src/packages/backend n +Welcome to Node.js v18.17.1. +Type ".help" for more information. > env = await require("@cocalc/backend/nats/env").getEnv() a = require("@cocalc/nats/sync/dstream"); s = await a.dstream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo', env}) @@ -68,6 +71,10 @@ export class DStream extends EventEmitter { return [...this.messages, ...Object.values(this.local)]; }; + get length() { + return this.messages.length; + } + publish = (mesg, subject?: string) => { const id = randomId(); this.local[id] = { mesg, subject }; diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index e5068404ea..4fe9969436 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -10,7 +10,8 @@ TODO: DEVELOPMENT: -~/cocalc/src/packages/server$ n +# note the package directory!! +~/cocalc/src/packages/backend n Welcome to Node.js v18.17.1. Type ".help" for more information. > env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = new a.Stream({name:'test',env,subjects:'foo',filter:'foo'}); await s.init(); @@ -179,6 +180,10 @@ export class Stream extends EventEmitter { return [...this.messages]; }; + get length() { + return this.messages.length; + } + publish = async (mesg: any, subject?: string, options?) => { if (this.js == null) { throw Error("closed"); @@ -430,6 +435,9 @@ export interface UserStreamOptions { } export function userStreamOptionsKey(options: UserStreamOptions) { + if (!options.name) { + throw Error("name must be specified"); + } const { env, ...x } = options; return JSON.stringify(x); } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index d60b9db1fe..4599d6ac6f 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -76,6 +76,9 @@ importers: '@cocalc/backend': specifier: workspace:* version: 'link:' + '@cocalc/nats': + specifier: workspace:* + version: link:../nats '@cocalc/util': specifier: workspace:* version: link:../util From 7c1793e90d75ac3fd21986a31805205bb6b62a69 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 8 Feb 2025 17:47:14 +0000 Subject: [PATCH 143/281] nats stream: ability to fill in old values --- src/packages/nats/sync/dstream.ts | 27 ++++++++-- src/packages/nats/sync/stream.ts | 89 +++++++++++++++++++++++-------- 2 files changed, 89 insertions(+), 27 deletions(-) diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index f5250250a2..f51a7a0cac 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -8,8 +8,7 @@ DEVELOPMENT: ~/cocalc/src/packages/backend n Welcome to Node.js v18.17.1. Type ".help" for more information. -> env = await require("@cocalc/backend/nats/env").getEnv() - a = require("@cocalc/nats/sync/dstream"); s = await a.dstream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo', env}) +> a = require("@cocalc/backend/nats/sync"); s = await a.dstream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo'}) */ @@ -67,12 +66,23 @@ export class DStream extends EventEmitter { delete this.raw; }; - get = () => { - return [...this.messages, ...Object.values(this.local)]; + get = (n?) => { + if (n == null) { + return [...this.messages, ...Object.values(this.local)]; + } else { + return ( + this.messages[n] ?? Object.values(this.local)[n - this.messages.length] + ); + } + }; + + // sequence number of n-th message + seq = (n) => { + return this.raw[n]?.seq; }; get length() { - return this.messages.length; + return this.messages.length + Object.keys(this.local).length; } publish = (mesg, subject?: string) => { @@ -124,6 +134,13 @@ export class DStream extends EventEmitter { // NOTE: ES6 spec guarantees "String keys are returned in the order in which they were added to the object." await awaitMap(Object.keys(this.local), MAX_PARALLEL, f); }); + + load = async (opts) => { + if (this.stream == null) { + throw Error("closed"); + } + await this.stream.load(opts); + }; } const dstreamCache: { [key: string]: DStream } = {}; diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 4fe9969436..75382b62ca 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -1,13 +1,6 @@ /* Consistent Centralized Event Stream = ordered list of messages -TODO: - - ability to easily initialize with only messages starting at a given seq - - load old messages starting at a given seq. - - maybe the limits and other config should be stored in a KV store so - they are sync'd between clients automatically. That's what NATS surely - does internally. - DEVELOPMENT: # note the package directory!! @@ -34,6 +27,11 @@ With browser client using a project: In browser: > s = await cc.client.nats_client.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo',limits:{max_msgs:5,max_age:1000000*1000*15,max_bytes:10000,max_msg_size:1000}}) +TODO: + - maybe the limits and other config should be stored in a KV store so + they are sync'd between clients automatically. That's what NATS surely + does internally. + */ @@ -176,8 +174,17 @@ export class Stream extends EventEmitter { this.startFetch(); }); - get = () => { - return [...this.messages]; + get = (n?) => { + if (n == null) { + return [...this.messages]; + } else { + return this.messages[n]; + } + }; + + // get sequence number of n-th message in stream + seq = (n) => { + return this.raw[n]?.seq; }; get length() { @@ -203,7 +210,9 @@ export class Stream extends EventEmitter { return resp; }; - private getConsumer = async ({ startSeq }: { startSeq?: number } = {}) => { + private getConsumer = async ({ start_seq }: { start_seq?: number } = {}) => { + // NOTE: do not cache or modify this in this function getConsumer, + // since it is also called by load and when reconnecting. const js = jetstream(this.env.nc); const jsm = await jetstreamManager(this.env.nc); // making an ephemeral consumer, which is automatically destroyed by NATS @@ -213,13 +222,13 @@ export class Stream extends EventEmitter { inactive_threshold: nanos(EPHEMERAL_CONSUMER_THRESH), }; let startOptions; - if (startSeq == null && this.start_seq != null) { - startSeq = this.start_seq; + if (start_seq == null && this.start_seq != null) { + start_seq = this.start_seq; } - if (startSeq != null) { + if (start_seq != null) { startOptions = { deliver_policy: "by_start_sequence", - opt_start_seq: startSeq, + opt_start_seq: start_seq, }; } else { startOptions = {}; @@ -284,8 +293,8 @@ export class Stream extends EventEmitter { // starting AFTER the last event we retrieved this.watch.stop(); // stop current watch // make new one: - const startSeq = this.raw[this.raw.length - 1]?.seq + 1; - this.startFetch({ startSeq }); + const start_seq = this.raw[this.raw.length - 1]?.seq + 1; + this.startFetch({ start_seq }); return; // because startFetch creates a new consumer monitor loop } } @@ -293,17 +302,21 @@ export class Stream extends EventEmitter { await delay(CONSUMER_MONITOR_INTERVAL); }; - private handle = (raw, noEmit = false) => { - let event; + private decode = (raw) => { try { - event = this.env.jc.decode(raw.data); + return this.env.jc.decode(raw.data); } catch { - event = raw.data; + // better than crashing + return raw.data; } - this.messages.push(event); + }; + + private handle = (raw, noEmit = false) => { + const mesg = this.decode(raw); + this.messages.push(mesg); this.raw.push(raw); if (!noEmit) { - this.emit("change", event, raw); + this.emit("change", mesg, raw); } }; @@ -420,6 +433,38 @@ export class Stream extends EventEmitter { ENFORCE_LIMITS_THROTTLE_MS, { leading: true, trailing: true }, ); + + // load older messages starting at start_seq + load = async ({ start_seq }: { start_seq: number }) => { + if (this.start_seq == null) { + // we already loaded everything on initialization; there can't be anything older. + return; + } + const consumer = await this.getConsumer({ start_seq }); + const info = await consumer.info(); + const fetch = await consumer.fetch(); + let i = 0; + // grab the messages. This should be very efficient since it + // internally grabs them in batches. + const raw: any[] = []; + const messages: any[] = []; + const cur = this.raw[0]?.seq; + for await (const x of fetch) { + if (cur != null && x.seq >= cur) { + break; + } + raw.push(x); + messages.push(this.decode(x)); + i += 1; + if (i >= info.num_pending) { + break; + } + } + // mutate the arrows this.raw and this.messages by splicing in + // raw and messages at the beginning: + this.raw.unshift(...raw); + this.messages.unshift(...messages); + }; } // One stream for each account and one for each project. From 88bde9a7d9e9102fcfc6819d45dc45c3ea5d7d05 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 8 Feb 2025 20:25:18 +0000 Subject: [PATCH 144/281] nats streams: s[n] notation; new public streams (hub can write; projects and accounts can only read) --- src/packages/frontend/nats/client.ts | 20 +++++++++----------- src/packages/nats/names.ts | 4 ++-- src/packages/nats/sync/dstream.ts | 23 ++++++++++++++++++++--- src/packages/nats/sync/stream.ts | 13 ++++++++++++- src/packages/server/nats/auth.ts | 11 ++++++----- src/packages/util/misc.ts | 12 ++++++++++++ 6 files changed, 61 insertions(+), 22 deletions(-) diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index ad565c007e..ad0820f89b 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -21,7 +21,7 @@ import type { ChatOptions } from "@cocalc/util/types/llm"; import { SystemKv } from "@cocalc/nats/system"; import { KV } from "@cocalc/nats/sync/kv"; import { DKV } from "@cocalc/nats/sync/dkv"; -import { stream } from "@cocalc/nats/sync/stream"; +import { stream, type UserStreamOptions } from "@cocalc/nats/sync/stream"; import { dstream } from "@cocalc/nats/sync/dstream"; import { initApi } from "@cocalc/frontend/nats/api"; import { delay } from "awaiting"; @@ -457,19 +457,17 @@ export class NatsClient { return dkv; }; - stream = async (opts: { - account_id?: string; - project_id?: string; - name: string; - }) => { + stream = async (opts: Partial) => { + if (!opts.account_id && !opts.project_id && opts.limits != null) { + throw Error("account client can't set limits on public stream"); + } return await stream({ ...opts, env: await this.getEnv() }); }; - dstream = async (opts: { - account_id?: string; - project_id?: string; - name: string; - }) => { + dstream = async (opts: Partial) => { + if (!opts.account_id && !opts.project_id && opts.limits != null) { + throw Error("account client can't set limits on public stream"); + } return await dstream({ ...opts, env: await this.getEnv() }); }; diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts index b0a87456a0..9b0c1f1e14 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/nats/names.ts @@ -36,7 +36,7 @@ export function jsName({ return `project-${project_id}`; } if (!account_id) { - throw Error("at least one of account_id and project_id must be set"); + return "public"; } return `account-${account_id}`; } @@ -55,7 +55,7 @@ export function streamSubject({ return `project.${project_id}.stream.>`; } if (!account_id) { - throw Error("at least one of account_id and project_id must be set"); + return "public.stream.>"; } return `account.${account_id}.stream.>`; } diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index f51a7a0cac..efd5074068 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -8,7 +8,11 @@ DEVELOPMENT: ~/cocalc/src/packages/backend n Welcome to Node.js v18.17.1. Type ".help" for more information. -> a = require("@cocalc/backend/nats/sync"); s = await a.dstream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo'}) + +> s = await require("@cocalc/backend/nats/sync").dstream({name:'test'}); + + +> s = await require("@cocalc/backend/nats/sync").dstream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo'});0 */ @@ -24,6 +28,8 @@ import { jsName, streamSubject, randomId } from "@cocalc/nats/names"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { delay } from "awaiting"; import { map as awaitMap } from "awaiting"; +import { isNumericString } from "@cocalc/util/misc"; + const MAX_PARALLEL = 50; export class DStream extends EventEmitter { @@ -37,6 +43,13 @@ export class DStream extends EventEmitter { this.stream = new Stream(opts); this.messages = this.stream.messages; this.raw = this.stream.raw; + return new Proxy(this, { + get(target, prop) { + return typeof prop == "string" && isNumericString(prop) + ? target.get(parseInt(prop)) + : target[String(prop)]; + }, + }); } init = reuseInFlight(async () => { @@ -68,10 +81,14 @@ export class DStream extends EventEmitter { get = (n?) => { if (n == null) { - return [...this.messages, ...Object.values(this.local)]; + return [ + ...this.messages, + ...Object.values(this.local).map((x) => x.mesg), + ]; } else { return ( - this.messages[n] ?? Object.values(this.local)[n - this.messages.length] + this.messages[n] ?? + Object.values(this.local)[n - this.messages.length]?.mesg ); } }; diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 75382b62ca..2e07e2f29f 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -7,6 +7,9 @@ DEVELOPMENT: ~/cocalc/src/packages/backend n Welcome to Node.js v18.17.1. Type ".help" for more information. +> s = await require("@cocalc/backend/nats/sync").stream({name:'test'}) + + > env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = new a.Stream({name:'test',env,subjects:'foo',filter:'foo'}); await s.init(); @@ -43,6 +46,7 @@ import { jsName, streamSubject } from "@cocalc/nats/names"; import { nanos, type Nanos } from "@cocalc/nats/util"; import { delay } from "awaiting"; import { throttle } from "lodash"; +import { isNumericString } from "@cocalc/util/misc"; // confirm that ephemeral consumer still exists every 15 seconds: // In case of a long disconnect from the network, this is what @@ -148,6 +152,13 @@ export class Stream extends EventEmitter { max_msg_size: -1, ...limits, }; + return new Proxy(this, { + get(target, prop) { + return typeof prop == "string" && isNumericString(prop) + ? target.get(parseInt(prop)) + : target[String(prop)]; + }, + }); } init = reuseInFlight(async () => { @@ -316,7 +327,7 @@ export class Stream extends EventEmitter { this.messages.push(mesg); this.raw.push(raw); if (!noEmit) { - this.emit("change", mesg, raw); + this.emit("change", mesg); } }; diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 4f9c99e517..9ef6cee391 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -117,10 +117,14 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { ]); const goalSub = new Set([ "_INBOX.>", // so can user request/response - //"$JS.API.>", // TODO! This needs to be restrained more, I think??! Don't know. "system.>", // access to READ the system info kv store. ]); + // the public jetstream: this makes it available *read only* to all accounts and projects. + goalPub.add("$JS.API.*.*.public"); + goalPub.add("$JS.API.*.*.public.>"); + goalPub.add("$JS.API.CONSUMER.MSG.NEXT.public.>"); + if (userType == "account") { goalSub.add(`*.account-${userId}.>`); goalPub.add(`*.account-${userId}.>`); @@ -131,7 +135,7 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { goalSub.add(`$SRV.*`); goalPub.add(`$SRV.*`); - // jetstream + // the account-specific kv stores goalPub.add(`$JS.API.*.*.KV_account-${userId}`); goalPub.add(`$JS.API.*.*.KV_account-${userId}.>`); @@ -144,9 +148,6 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { add(goalSub, sub); add(goalPub, pub); } - // TODO: there will be other subjects - // TODO: something similar for projects, e.g., they can publish to a channel that browser clients - // will listen to, e.g., for timetravel editing. } else if (userType == "project") { // microservices api goalSub.add(`$SRV.*.project-${userId}.>`); diff --git a/src/packages/util/misc.ts b/src/packages/util/misc.ts index 454a6d8920..b39a590374 100644 --- a/src/packages/util/misc.ts +++ b/src/packages/util/misc.ts @@ -2695,3 +2695,15 @@ export function basePathCookieName({ }): string { return `${basePath.length <= 1 ? "" : encodeURIComponent(basePath)}${name}`; } + +export function isNumericString(str: string): boolean { + // https://stackoverflow.com/questions/175739/how-can-i-check-if-a-string-is-a-valid-number + if (typeof str != "string") { + return false; // we only process strings! + } + return ( + // @ts-ignore + !isNaN(str) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)... + !isNaN(parseFloat(str)) + ); // ...and ensure strings of whitespace fail +} From 017067e1d36f30a34302bdaad1e1d48cbacfd609 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 8 Feb 2025 22:58:34 +0000 Subject: [PATCH 145/281] nats: user friendly account/project/public kv an dkv --- src/packages/backend/nats/sync.ts | 10 + src/packages/frontend/nats/client.ts | 98 ++----- src/packages/nats/sync/dkv.ts | 304 ++++++--------------- src/packages/nats/sync/dstream.ts | 9 + src/packages/nats/sync/general-dkv.ts | 276 +++++++++++++++++++ src/packages/nats/sync/general-kv.ts | 328 ++++++++++++++++++++++ src/packages/nats/sync/kv.ts | 380 +++++++------------------- src/packages/nats/sync/stream.ts | 7 + src/packages/nats/system.ts | 4 +- 9 files changed, 825 insertions(+), 591 deletions(-) create mode 100644 src/packages/nats/sync/general-dkv.ts create mode 100644 src/packages/nats/sync/general-kv.ts diff --git a/src/packages/backend/nats/sync.ts b/src/packages/backend/nats/sync.ts index 5f868ab5bd..8e4c614682 100644 --- a/src/packages/backend/nats/sync.ts +++ b/src/packages/backend/nats/sync.ts @@ -1,5 +1,7 @@ import { stream as createStream } from "@cocalc/nats/sync/stream"; import { dstream as createDstream } from "@cocalc/nats/sync/dstream"; +import { kv as createKV } from "@cocalc/nats/sync/kv"; +import { dkv as createDKV } from "@cocalc/nats/sync/dkv"; import { getEnv } from "@cocalc/backend/nats/env"; export async function stream(opts) { @@ -9,3 +11,11 @@ export async function stream(opts) { export async function dstream(opts) { return await createDstream({ ...opts, env: await getEnv() }); } + +export async function kv(opts) { + return await createKV({ ...opts, env: await getEnv() }); +} + +export async function dkv(opts) { + return await createDKV({ ...opts, env: await getEnv() }); +} diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index ad0820f89b..51f0753921 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -6,7 +6,7 @@ import { join } from "path"; import * as jetstream from "@nats-io/jetstream"; import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; import { randomId } from "@cocalc/nats/names"; -import { browserSubject, projectSubject, jsName } from "@cocalc/nats/names"; +import { browserSubject, projectSubject } from "@cocalc/nats/names"; import { parse_query } from "@cocalc/sync/table/util"; import { sha1 } from "@cocalc/util/misc"; import { keys } from "lodash"; @@ -18,9 +18,8 @@ import { isValidUUID } from "@cocalc/util/misc"; import { OpenFiles } from "@cocalc/nats/sync/open-files"; import { PubSub } from "@cocalc/nats/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; -import { SystemKv } from "@cocalc/nats/system"; -import { KV } from "@cocalc/nats/sync/kv"; -import { DKV } from "@cocalc/nats/sync/dkv"; +import { kv, type KVOptions } from "@cocalc/nats/sync/kv"; +import { dkv, type DKVOptions } from "@cocalc/nats/sync/dkv"; import { stream, type UserStreamOptions } from "@cocalc/nats/sync/stream"; import { dstream } from "@cocalc/nats/sync/dstream"; import { initApi } from "@cocalc/frontend/nats/api"; @@ -38,7 +37,6 @@ export class NatsClient { public hub: HubApi; public sessionId = randomId(); private openFilesCache: { [project_id: string]: OpenFiles } = {}; - private theSystemKv?: SystemKv; constructor(client: WebappClient) { this.client = client; @@ -385,90 +383,32 @@ export class NatsClient { return accumulate; }; - systemKv = reuseInFlight(async () => { - if (this.theSystemKv == null) { - const s = new SystemKv(await this.getEnv()); - await s.init(); - this.theSystemKv = s; - } - return this.theSystemKv!; - }); - - private kvCache: { [key: string]: KV } = {}; - kv = reuseInFlight( - async ({ - account_id, - project_id, - filter, - options, - }: { - account_id?: string; - project_id?: string; - filter?: string | string[]; - options?; - } = {}) => { - if (!account_id && !project_id) { - account_id = this.client.account_id; - } - const name = jsName({ account_id, project_id }); - const key = JSON.stringify([name, filter, options]); - if (this.kvCache[key] == null) { - const kv = new KV({ - name, - filter, - options, - env: await this.getEnv(), - }); - await kv.init(); - this.kvCache[key] = kv; - kv.on("closed", () => { - delete this.kvCache[key]; - }); - } - return this.kvCache[key]; - }, - ); - - dkv = async ({ - account_id, - project_id, - filter, - options, - merge, - }: { - account_id?: string; - project_id?: string; - filter: string | string[]; - options?; - merge?: (opts: { ancestor?; local?; remote?; key?: string }) => any; - }) => { - if (!account_id && !project_id) { - account_id = this.client.account_id; - } - const name = jsName({ account_id, project_id }); - const dkv = new DKV({ - name, - filter, - options, - merge, - env: await this.getEnv(), - }); - await dkv.init(); - return dkv; - }; - stream = async (opts: Partial) => { if (!opts.account_id && !opts.project_id && opts.limits != null) { throw Error("account client can't set limits on public stream"); } - return await stream({ ...opts, env: await this.getEnv() }); + return await stream({ env: await this.getEnv(), ...opts }); }; dstream = async (opts: Partial) => { if (!opts.account_id && !opts.project_id && opts.limits != null) { throw Error("account client can't set limits on public stream"); } - return await dstream({ ...opts, env: await this.getEnv() }); + return await dstream({ env: await this.getEnv(), ...opts }); + }; + + kv = async (opts: Partial) => { + // if (!opts.account_id && !opts.project_id && opts.limits != null) { + // throw Error("account client can't set limits on public stream"); + // } + return await kv({ env: await this.getEnv(), ...opts }); + }; + + dkv = async (opts: Partial) => { + // if (!opts.account_id && !opts.project_id && opts.limits != null) { + // throw Error("account client can't set limits on public stream"); + // } + return await dkv({ env: await this.getEnv(), ...opts }); }; microservicesClient = async () => { diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index a253f07dea..ae5fbb2bed 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -1,270 +1,126 @@ /* -Eventually Consistent Distributed Key:Value Store - -- You give one or more subjects and this provides a synchronous eventually consistent - "multimaster" distributed way to work with the KV store of keys matching any of those subjects, - inside of the named KV store. -- You should define a 3-way merge function, which is used to automatically resolve all - conflicting writes. The default is to use our local version, i.e., "last write to remote wins". -- All set/get/delete operations are synchronous. -- The state gets sync'd in the backend to NATS as soon as possible. - -This class is based on top of the Consistent Centralized Key:Value Store defined in kv.ts. -You can use the same key:value store at the same time via both interfaces, and if store -is a DKV, you can also access the underlying KV via "store.kv". - -- You must explicitly call "await store.init()" to initialize this before using it. - -- The store emits an event ('change', key) whenever anything changes. - -- Calling "store.get()" provides ALL the data, and "store.get(key)" gets one value. - -- Use "store.set(key,value)" or "store.set({key:value, key2:value2, ...})" to set data, - with the following semantics: - - - in the background, changes propagate to NATS. You do not do anything explicitly and - this should never raise an exception. - - - you can call "store.hasUnsavedChanges()" to see if there are any unsaved changes. - - - call "store.unsavedChanges()" to see the unsaved keys. - -- The 3-way merge function takes as input {local,remote,ancestor,key}, where - - key = the key where there's a conflict - - local = your version of the value - - remote = the remote value, which conflicts in that isEqual(local,remote) is false. - - ancestor = a known common ancestor of local and remote. - - (any of local, remote or ancestor can be undefined, e.g., no previous value or a key was deleted) - - You can do anything synchronously you want to resolve such conflicts, i.e., there are no - axioms that have to be satisifed. If the 3-way merge function throws an exception (or is - not specified) we silently fall back to "last write wins". +Always Consistent Centralized Key Value Store DEVELOPMENT: -~/cocalc/src/packages/server$ node +~/cocalc/src/packages/backend n Welcome to Node.js v18.17.1. Type ".help" for more information. -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/dkv"); s = new a.DKV({name:'test',env,filter:['foo.>'],merge:({local,remote})=>{return {...remote,...local}}}); await s.init(); - - -In the browser console: +> t = await require("@cocalc/backend/nats/sync").dkv({name:'test'}) -> s = await cc.client.nats_client.dkv({filter:['foo.>'],merge:({local,remote})=>{return {...remote,...local}}}) - -# NOTE that the name is account-{account_id} or project-{project_id}, -# and if not given defaults to the account-{user's account id} -> s.kv.name -'account-6aae57c6-08f1-4bb5-848b-3ceb53e61ede' - -> s.on('change',(key)=>console.log(key));0; - - -TODO: - - require not-everything subject or have an explicit size limit? */ import { EventEmitter } from "events"; -import { KV } from "./kv"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { type NatsEnv } from "@cocalc/nats/types"; -import { isEqual } from "lodash"; -import { delay } from "awaiting"; +import { GeneralDKV, type MergeFunction } from "./general-dkv"; +import { userKvKey, type KVOptions } from "./kv"; +import { jsName } from "@cocalc/nats/names"; -const TOMBSTONE = Symbol("tombstone"); +export interface DKVOptions extends KVOptions { + merge: MergeFunction; +} export class DKV extends EventEmitter { - private kv?: KV; - private merge?: (opts: { - key: string; - ancestor: any; - local: any; - remote: any; - }) => any; - private local: { [key: string]: any } = {}; - private changed: Set = new Set(); + generalDKV?: GeneralDKV; + name: string; - constructor({ - name, - env, - filter, - merge, - options, - }: { - name: string; - env: NatsEnv; - // 3-way merge conflict resolution - merge?: (opts: { - key: string; - ancestor?: any; - local?: any; - remote?: any; - }) => any; - // filter: optionally restrict to subset of named kv store matching these subjects. - // NOTE: any key name that you *set or delete* must match one of these - filter: string | string[]; - options?; - }) { + constructor({ name, account_id, project_id, merge, env }: DKVOptions) { super(); - this.merge = merge; - this.kv = new KV({ name, env, filter, options }); + // name of the jetstream key:value store. + const kvname = jsName({ account_id, project_id }); + this.name = name; + this.generalDKV = new GeneralDKV({ + name: kvname, + filter: `${name}.>`, + env, + merge, + }); + this.init(); + return new Proxy(this, { + deleteProperty(target, prop) { + target.delete(prop); + return true; + }, + set(target, prop, value) { + prop = String(prop); + if (prop == "_eventsCount") { + target[prop] = value; + return true; + } + if (target[prop] != null) { + throw Error(`method name '${prop}' is read only`); + } + target.set(prop, value); + return true; + }, + get(target, prop) { + return target[String(prop)] ?? target.get(String(prop)); + }, + }); } init = reuseInFlight(async () => { - if (this.kv == null) { + if (this.generalDKV == null) { throw Error("closed"); } - this.kv.on("change", this.handleRemoteChange); - await this.kv.init(); - this.emit("connected"); + await this.generalDKV.init(); }); close = () => { - if (this.kv == null) { + if (this.generalDKV == null) { return; } - this.kv.close(); + this.generalDKV.close(); + delete this.generalDKV; this.emit("closed"); this.removeAllListeners(); - delete this.kv; - // @ts-ignore - delete this.local; - // @ts-ignore - delete this.changed; - delete this.merge; }; - private handleRemoteChange = (key, remote, ancestor) => { - const local = this.local[key]; - if (local !== undefined) { - if (isEqual(local, remote)) { - // we have a local change, but it's the same change as remote, so just - // forget about our local change. - delete this.local[key]; - } else { - let value; - try { - value = this.merge?.({ key, local, remote, ancestor }); - } catch { - // user provided a merge function that throws an exception. We select local, since - // it is the newest, i.e., "last write wins" - value = local; - } - if (isEqual(value, remote)) { - // no change, so forget our local value - delete this.local[key]; - } else { - // resolve with the new value, or if it is undefined, a TOMBSTONE, meaning choice is to delete. - this.local[key] = value ?? TOMBSTONE; - } - } + delete = (key) => { + if (this.generalDKV == null) { + throw Error("closed"); } - this.emit("change", key); + this.generalDKV.delete(`${this.name}.${key}`); }; get = (key?) => { - if (this.kv == null) { + if (this.generalDKV == null) { throw Error("closed"); } - if (key != null) { - this.assertValidKey(key); - const local = this.local[key]; - if (local === TOMBSTONE) { - return undefined; - } - return local ?? this.kv.get(key); - } - const x = { ...this.kv.get(), ...this.local }; - for (const key in this.local) { - if (this.local[key] === TOMBSTONE) { - delete x[key]; + if (key == null) { + const obj = this.generalDKV.get(); + const x: any = {}; + for (const k in obj) { + x[k.slice(this.name.length + 1)] = obj[k]; } - } - return x; - }; - - private assertValidKey = (key) => { - if (this.kv == null) { - throw Error("closed"); - } - this.kv.assertValidKey(key); - }; - - delete = (key) => { - this.assertValidKey(key); - this.local[key] = TOMBSTONE; - this.changed.add(key); - this.save(); - }; - - set = (...args) => { - if (args.length == 2) { - this.assertValidKey(args[0]); - this.local[args[0]] = args[1] ?? TOMBSTONE; - this.changed.add(args[0]); + return x; } else { - const obj = args[0]; - for (const key in obj) { - this.assertValidKey(key); - this.local[key] = obj[key] ?? TOMBSTONE; - this.changed.add(key); - } + return this.generalDKV.get(`${this.name}.${key}`); } - this.save(); }; - hasUnsavedChanges = () => { - if (this.kv == null) { - return false; + set = (key: string, value: any) => { + if (this.generalDKV == null) { + throw Error("closed"); } - return this.changed.size > 0 || Object.keys(this.local).length > 0; - }; - - unsavedChanges = () => { - return Object.keys(this.local); + this.generalDKV.set(`${this.name}.${key}`, value); }; +} - private save = reuseInFlight(async () => { - let d = 100; - while (true) { - try { - await this.attemptToSave(); - //console.log("successfully saved"); - } catch { - //(err) { - // console.log("problem saving", err); - } - if (this.hasUnsavedChanges()) { - d = Math.min(10000, d * 1.3) + Math.random() * 100; - await delay(d); - } else { - return; - } - } - }); - - private attemptToSave = reuseInFlight(async () => { - if (this.kv == null) { - throw Error("closed"); +const cache: { [key: string]: DKV } = {}; +export const dkv = reuseInFlight( + async (opts: DKVOptions) => { + const key = userKvKey(opts); + if (cache[key] == null) { + const k = new DKV(opts); + await k.init(); + k.on("closed", () => delete cache[key]); + cache[key] = k; } - this.changed.clear(); - const obj = { ...this.local }; - for (const key in obj) { - if (obj[key] === TOMBSTONE) { - await this.kv.delete(key); - delete obj[key]; - if (!this.changed.has(key)) { - delete this.local[key]; - } - } - } - await this.kv.set(obj); - for (const key in obj) { - if (obj[key] === this.local[key] && !this.changed.has(key)) { - delete this.local[key]; - } - } - }); -} + return cache[key]!; + }, + { + createKey: (args) => userKvKey(args[0]), + }, +); diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index efd5074068..9c84f9746b 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -108,6 +108,15 @@ export class DStream extends EventEmitter { this.save(); }; + push = (...args) => { + if (this.stream == null) { + throw Error("closed"); + } + for (const mesg of args) { + this.publish(mesg); + } + }; + hasUnsavedChanges = () => { if (this.stream == null) { return false; diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts new file mode 100644 index 0000000000..d368013775 --- /dev/null +++ b/src/packages/nats/sync/general-dkv.ts @@ -0,0 +1,276 @@ +/* +Eventually Consistent Distributed Key:Value Store + +- You give one or more subjects and this provides a synchronous eventually consistent + "multimaster" distributed way to work with the KV store of keys matching any of those subjects, + inside of the named KV store. +- You should define a 3-way merge function, which is used to automatically resolve all + conflicting writes. The default is to use our local version, i.e., "last write to remote wins". +- All set/get/delete operations are synchronous. +- The state gets sync'd in the backend to NATS as soon as possible. + +This class is based on top of the Consistent Centralized Key:Value Store defined in kv.ts. +You can use the same key:value store at the same time via both interfaces, and if store +is a DKV, you can also access the underlying KV via "store.kv". + +- You must explicitly call "await store.init()" to initialize this before using it. + +- The store emits an event ('change', key) whenever anything changes. + +- Calling "store.get()" provides ALL the data, and "store.get(key)" gets one value. + +- Use "store.set(key,value)" or "store.set({key:value, key2:value2, ...})" to set data, + with the following semantics: + + - in the background, changes propagate to NATS. You do not do anything explicitly and + this should never raise an exception. + + - you can call "store.hasUnsavedChanges()" to see if there are any unsaved changes. + + - call "store.unsavedChanges()" to see the unsaved keys. + +- The 3-way merge function takes as input {local,remote,ancestor,key}, where + - key = the key where there's a conflict + - local = your version of the value + - remote = the remote value, which conflicts in that isEqual(local,remote) is false. + - ancestor = a known common ancestor of local and remote. + + (any of local, remote or ancestor can be undefined, e.g., no previous value or a key was deleted) + + You can do anything synchronously you want to resolve such conflicts, i.e., there are no + axioms that have to be satisifed. If the 3-way merge function throws an exception (or is + not specified) we silently fall back to "last write wins". + + +DEVELOPMENT: + +~/cocalc/src/packages/server$ node +Welcome to Node.js v18.17.1. +Type ".help" for more information. +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/dkv"); s = new a.DKV({name:'test',env,filter:['foo.>'],merge:({local,remote})=>{return {...remote,...local}}}); await s.init(); + + +In the browser console: + +> s = await cc.client.nats_client.dkv({filter:['foo.>'],merge:({local,remote})=>{return {...remote,...local}}}) + +# NOTE that the name is account-{account_id} or project-{project_id}, +# and if not given defaults to the account-{user's account id} +> s.kv.name +'account-6aae57c6-08f1-4bb5-848b-3ceb53e61ede' + +> s.on('change',(key)=>console.log(key));0; + + +TODO: + - require not-everything subject or have an explicit size limit? + - some history would be VERY useful here due to the merge conflicts. + - for conflict resolution maybe instead of local and remote, just give + two values along with their assigned sequence numbers (?). I.e., something + where the resolution doesn't depend on where it is run. ? Or maybe this doesn't matter. +*/ + +import { EventEmitter } from "events"; +import { GeneralKV } from "./general-kv"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { type NatsEnv } from "@cocalc/nats/types"; +import { isEqual } from "lodash"; +import { delay } from "awaiting"; + +const TOMBSTONE = Symbol("tombstone"); + +export type MergeFunction = (opts: { + key: string; + ancestor: any; + local: any; + remote: any; +}) => any; + +export class GeneralDKV extends EventEmitter { + private kv?: GeneralKV; + private merge?: MergeFunction; + private local: { [key: string]: any } = {}; + private changed: Set = new Set(); + + constructor({ + name, + env, + filter, + merge, + options, + }: { + name: string; + env: NatsEnv; + // 3-way merge conflict resolution + merge?: (opts: { + key: string; + ancestor?: any; + local?: any; + remote?: any; + }) => any; + // filter: optionally restrict to subset of named kv store matching these subjects. + // NOTE: any key name that you *set or delete* must match one of these + filter: string | string[]; + options?; + }) { + super(); + this.merge = merge; + this.kv = new GeneralKV({ name, env, filter, options }); + } + + init = reuseInFlight(async () => { + if (this.kv == null) { + throw Error("closed"); + } + this.kv.on("change", this.handleRemoteChange); + await this.kv.init(); + this.emit("connected"); + }); + + close = () => { + if (this.kv == null) { + return; + } + this.kv.close(); + this.emit("closed"); + this.removeAllListeners(); + delete this.kv; + // @ts-ignore + delete this.local; + // @ts-ignore + delete this.changed; + delete this.merge; + }; + + private handleRemoteChange = (key, remote, ancestor) => { + const local = this.local[key]; + if (local !== undefined) { + if (isEqual(local, remote)) { + // we have a local change, but it's the same change as remote, so just + // forget about our local change. + delete this.local[key]; + } else { + let value; + try { + value = this.merge?.({ key, local, remote, ancestor }); + } catch { + // user provided a merge function that throws an exception. We select local, since + // it is the newest, i.e., "last write wins" + value = local; + } + if (isEqual(value, remote)) { + // no change, so forget our local value + delete this.local[key]; + } else { + // resolve with the new value, or if it is undefined, a TOMBSTONE, meaning choice is to delete. + this.local[key] = value ?? TOMBSTONE; + } + } + } + this.emit("change", key); + }; + + get = (key?) => { + if (this.kv == null) { + throw Error("closed"); + } + if (key != null) { + this.assertValidKey(key); + const local = this.local[key]; + if (local === TOMBSTONE) { + return undefined; + } + return local ?? this.kv.get(key); + } + const x = { ...this.kv.get(), ...this.local }; + for (const key in this.local) { + if (this.local[key] === TOMBSTONE) { + delete x[key]; + } + } + return x; + }; + + private assertValidKey = (key) => { + if (this.kv == null) { + throw Error("closed"); + } + this.kv.assertValidKey(key); + }; + + delete = (key) => { + this.assertValidKey(key); + this.local[key] = TOMBSTONE; + this.changed.add(key); + this.save(); + }; + + set = (...args) => { + if (args.length == 2) { + this.assertValidKey(args[0]); + this.local[args[0]] = args[1] ?? TOMBSTONE; + this.changed.add(args[0]); + } else { + const obj = args[0]; + for (const key in obj) { + this.assertValidKey(key); + this.local[key] = obj[key] ?? TOMBSTONE; + this.changed.add(key); + } + } + this.save(); + }; + + hasUnsavedChanges = () => { + if (this.kv == null) { + return false; + } + return this.changed.size > 0 || Object.keys(this.local).length > 0; + }; + + unsavedChanges = () => { + return Object.keys(this.local); + }; + + private save = reuseInFlight(async () => { + let d = 100; + while (true) { + try { + await this.attemptToSave(); + //console.log("successfully saved"); + } catch { + //(err) { + // console.log("problem saving", err); + } + if (this.hasUnsavedChanges()) { + d = Math.min(10000, d * 1.3) + Math.random() * 100; + await delay(d); + } else { + return; + } + } + }); + + private attemptToSave = reuseInFlight(async () => { + if (this.kv == null) { + throw Error("closed"); + } + this.changed.clear(); + const obj = { ...this.local }; + for (const key in obj) { + if (obj[key] === TOMBSTONE) { + await this.kv.delete(key); + delete obj[key]; + if (!this.changed.has(key)) { + delete this.local[key]; + } + } + } + await this.kv.set(obj); + for (const key in obj) { + if (obj[key] === this.local[key] && !this.changed.has(key)) { + delete this.local[key]; + } + } + }); +} diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts new file mode 100644 index 0000000000..9f0d29ff9a --- /dev/null +++ b/src/packages/nats/sync/general-kv.ts @@ -0,0 +1,328 @@ +/* +Always Consistent Centralized Key Value Store + +- You give one or more subjects and this provides an asynchronous but consistent + way to work with the KV store of keys matching any of those subjects, + inside of the named KV store. +- The get operation is sync. (It can of course be slightly out of date, but that is detected + if you try to immediately write it.) +- The set will fail if the local cached value (returned by get) turns out to be out of date. +- Also delete and set will fail if the NATS connection is down or times out. +- For an eventually consistent sync wrapper around this, use DKV, defined in the sibling file dkv.ts. + +This is a simple KV wrapper around NATS's KV, for small KV stores. Each client holds a local cache +of all data, which is used to ensure set's are a no-op if there is no change. Also, this automates +ensuring that if you do a read-modify-write, this will succeed only if nobody else makes a change +before you. + +- You must explicitly call "await store.init()" to initialize it before using it. + +- The store emits an event ('change', key, newValue, previousValue) whenever anything changes + +- Calling "store.get()" provides ALL the data and is synchronous. It uses various API tricks to + ensure this is fast and is updated when there is any change from upstream. Use "store.get(key)" + to get the value of one key. + +- Use "await store.set(key,value)" or "await store.set({key:value, key2:value2, ...})" to set data, + with the following semantics: + + - set ONLY makes a change if our local version ("store.get(key)") of the value is different from + what you're trying to set the value to, where different is defined by lodash isEqual. + + - if our local version this.get(key) was not the most recent version in NATS, then the set will + definitely throw an exception! This is fantastic because it means you can modify and save what + is in the local cache on multiple nodes at once anywhere, and be 100% certain to never overwrite + data in complicated objects. Of course, you have to assume "await store.set(...)" WILL + sometimes fail. + + - Set with multiple keys "pipelines" in that MAX_PARALLEL key/value pairs are set at once, without + waiting for every single individual set to get ACK'd from the server before doing more sets. + This makes this **massively** faster, but means that if "await store.set(...)" fails, you don't + immediately know which keys were successfully set and which failed, though all keys worked will get + updated soon and reflected in store.get(). + +- Use "await store.expire(ageMs)" to delete every key that was last changed at least ageMs + milliseconds in the past. + + TODO/WARNING: the timestamps are defined by NATS (and its clock), but + the definition of "ageMs in the past" is defined by the client where this is called. Thus + if the client's clock is off, that would be a huge problem. An obvious solution is to + get the current time from NATS, and use that. I don't know a "good" way to get the current + time except maybe publishing a message to myself...? + +TODO: + +- [ ] maybe expose some functionality related to versions/history? + +DEVELOPMENT: + +~/cocalc/src/packages/server$ n +Welcome to Node.js v18.17.1. +Type ".help" for more information. +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/general-kv"); s = new a.GeneralKV({name:'test',env,filter:['foo.>']}); await s.init(); + +> await s.set({"foo.x":10}) // or s.set("foo.x", 10) +> s.get() +{ 'foo.x': 10 } +> await s.delete("foo.x") +undefined +> s.get() +{} +> await s.set({"foo.x":10, "foo.bar":20}) + +// Since the filters are disjoint these are totally different: + +> t = new a.KV({name:'test',env,filter:['bar.>']}); await t.init(); +> await t.get() +{} +> await t.set({"bar.abc":10}) +undefined +> await t.get() +{ 'bar.abc': Uint8Array(2) [ 49, 48 ] } +> await s.get() +{ 'foo.x': 10, 'foo.bar': 20, 'bar.abc': 10 } + +// The union: +> u = new a.KV({name:'test',env,filter:['bar.>', 'foo.>']}); await u.init(); +> u.get() +{ 'foo.x': 10, 'foo.bar': 20, 'bar.abc': 10 } +> await s.set({'foo.x':999}) +undefined +> u.get() +{ 'foo.x': 999, 'foo.bar': 20, 'bar.abc': 10 } +*/ + +import { EventEmitter } from "events"; +import { type NatsEnv } from "@cocalc/nats/types"; +import { Kvm } from "@nats-io/kv"; +import { getAllFromKv, matchesPattern } from "@cocalc/nats/util"; +import { isEqual } from "lodash"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { map as awaitMap } from "awaiting"; + +const MAX_PARALLEL = 50; + +export class GeneralKV extends EventEmitter { + public readonly name: string; + private options?; + private filter?: string[]; + private env: NatsEnv; + private kv?; + private watch?; + private all?: { [key: string]: any }; + private revisions?: { [key: string]: number }; + private times?: { [key: string]: Date }; + + constructor({ + name, + env, + filter, + options, + }: { + name: string; + // filter: optionally restrict to subset of named kv store matching these subjects. + // NOTE: any key name that you *set or delete* should match one of these + filter?: string | string[]; + env: NatsEnv; + options?; + }) { + super(); + this.env = env; + this.name = name; + this.options = options; + this.filter = typeof filter == "string" ? [filter] : filter; + } + + init = reuseInFlight(async () => { + if (this.all != null) { + return; + } + const kvm = new Kvm(this.env.nc); + this.kv = await kvm.create(this.name, { + compression: true, + ...this.options, + }); + const { all, revisions, times } = await getAllFromKv({ + kv: this.kv, + key: this.filter, + }); + this.revisions = revisions; + this.times = times; + for (const key in all) { + all[key] = this.env.jc.decode(all[key]); + } + this.all = all; + this.emit("connected"); + this.startWatch(); + }); + + private startWatch = async () => { + // watch for changes + this.watch = await this.kv.watch({ + ignoreDeletes: false, + include: "updates", + key: this.filter, + }); + for await (const x of this.watch) { + const { revision, key, value, sm } = x; + if (this.revisions == null || this.all == null || this.times == null) { + return; + } + this.revisions[key] = revision; + const prev = this.all[key]; + if (value.length == 0) { + // delete + delete this.all[key]; + delete this.times[key]; + } else { + this.all[key] = this.env.jc.decode(value); + this.times[key] = sm.time; + } + this.emit("change", key, this.all[key], prev); + } + }; + + close = () => { + this.watch?.stop(); + delete this.all; + delete this.times; + delete this.revisions; + this.emit("closed"); + this.removeAllListeners(); + }; + + get = (key?) => { + if (this.all == null) { + throw Error("not initialized"); + } + if (key == undefined) { + return { ...this.all }; + } else { + return this.all?.[key]; + } + }; + + time = (key?) => { + if (key == null) { + return this.times; + } else { + return this.times?.[key]; + } + }; + + assertValidKey = (key: string) => { + if (!this.isValidKey(key)) { + throw Error( + `delete: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, + ); + } + }; + + isValidKey = (key: string) => { + if (this.filter == null) { + return true; + } + for (const pattern of this.filter) { + if (matchesPattern({ pattern, subject: key })) { + return true; + } + } + return false; + }; + + delete = async (key, revision?) => { + this.assertValidKey(key); + if (this.all == null || this.revisions == null || this.times == null) { + throw Error("not ready"); + } + if (this.all[key] !== undefined) { + const cur = this.all[key]; + try { + delete this.all[key]; + const newRevision = await this.kv.delete(key, { + previousSeq: revision ?? this.revisions[key], + }); + this.revisions[key] = newRevision; + delete this.times[key]; + } catch (err) { + if (cur === undefined) { + delete this.all[key]; + } else { + this.all[key] = cur; + } + throw err; + } + } + }; + + // delete everything matching the filter that hasn't been set + // in the given amount of ms. Returns number of deleted records. + // NOTE: This could throw an exception if something that would expire + // were changed right when this is run then it would get expired + // but shouldn't. In that case, run it again. + expire = async (ageMs: number): Promise => { + if (!ageMs) { + throw Error("ageMs must be set"); + } + if (this.times == null || this.all == null) { + throw Error("not initialized"); + } + const cutoff = new Date(Date.now() - ageMs); + // make copy of revisions *before* we start deleting so that + // if a key is changed exactly while deleting we get an error + // and don't accidently delete it! + const revisions = { ...this.revisions }; + const toDelete = Object.keys(this.all).filter( + (key) => this.times?.[key] != null && this.times[key] <= cutoff, + ); + if (toDelete.length > 0) { + await awaitMap(toDelete, MAX_PARALLEL, async (key) => { + await this.delete(key, revisions[key]); + }); + } + return toDelete.length; + }; + + // delete all that we know about + clear = async () => { + if (this.all == null) { + throw Error("not initialized"); + } + await awaitMap(Object.keys(this.all), MAX_PARALLEL, this.delete); + }; + + set = async (...args) => { + if (args.length == 2) { + await this.setOne(args[0], args[1]); + return; + } + const obj = args[0]; + await awaitMap( + Object.keys(obj), + MAX_PARALLEL, + async (key) => await this.setOne(key, obj[key]), + ); + }; + + private setOne = async (key, value) => { + if (!this.isValidKey(key)) { + throw Error( + `set: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, + ); + } + if (this.all == null || this.revisions == null) { + throw Error("not ready"); + } + if (isEqual(this.all[key], value)) { + return; + } + if (value === undefined) { + return await this.delete(key); + } + const revision = this.revisions[key]; + const val = this.env.jc.encode(value); + await this.kv.put(key, val, { + previousSeq: revision, + }); + }; +} diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index c42e0cc2f6..e25fe3b3db 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -1,328 +1,136 @@ /* Always Consistent Centralized Key Value Store -- You give one or more subjects and this provides an asynchronous but consistent - way to work with the KV store of keys matching any of those subjects, - inside of the named KV store. -- The get operation is sync. (It can of course be slightly out of date, but that is detected - if you try to immediately write it.) -- The set will fail if the local cached value (returned by get) turns out to be out of date. -- Also delete and set will fail if the NATS connection is down or times out. -- For an eventually consistent sync wrapper around this, use DKV, defined in the sibling file dkv.ts. - -This is a simple KV wrapper around NATS's KV, for small KV stores. Each client holds a local cache -of all data, which is used to ensure set's are a no-op if there is no change. Also, this automates -ensuring that if you do a read-modify-write, this will succeed only if nobody else makes a change -before you. - -- You must explicitly call "await store.init()" to initialize it before using it. - -- The store emits an event ('change', key, newValue, previousValue) whenever anything changes - -- Calling "store.get()" provides ALL the data and is synchronous. It uses various API tricks to - ensure this is fast and is updated when there is any change from upstream. Use "store.get(key)" - to get the value of one key. - -- Use "await store.set(key,value)" or "await store.set({key:value, key2:value2, ...})" to set data, - with the following semantics: - - - set ONLY makes a change if our local version ("store.get(key)") of the value is different from - what you're trying to set the value to, where different is defined by lodash isEqual. - - - if our local version this.get(key) was not the most recent version in NATS, then the set will - definitely throw an exception! This is fantastic because it means you can modify and save what - is in the local cache on multiple nodes at once anywhere, and be 100% certain to never overwrite - data in complicated objects. Of course, you have to assume "await store.set(...)" WILL - sometimes fail. - - - Set with multiple keys "pipelines" in that MAX_PARALLEL key/value pairs are set at once, without - waiting for every single individual set to get ACK'd from the server before doing more sets. - This makes this **massively** faster, but means that if "await store.set(...)" fails, you don't - immediately know which keys were successfully set and which failed, though all keys worked will get - updated soon and reflected in store.get(). - -- Use "await store.expire(ageMs)" to delete every key that was last changed at least ageMs - milliseconds in the past. - - TODO/WARNING: the timestamps are defined by NATS (and its clock), but - the definition of "ageMs in the past" is defined by the client where this is called. Thus - if the client's clock is off, that would be a huge problem. An obvious solution is to - get the current time from NATS, and use that. I don't know a "good" way to get the current - time except maybe publishing a message to myself...? - -TODO: - -- [ ] maybe expose some functionality related to versions/history? DEVELOPMENT: -~/cocalc/src/packages/server$ n +~/cocalc/src/packages/backend n Welcome to Node.js v18.17.1. Type ".help" for more information. -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/kv"); s = new a.KV({name:'test',env,filter:['foo.>']}); await s.init(); - -> await s.set({"foo.x":10}) // or s.set("foo.x", 10) -> s.get() -{ 'foo.x': 10 } -> await s.delete("foo.x") -undefined -> s.get() -{} -> await s.set({"foo.x":10, "foo.bar":20}) - -// Since the filters are disjoint these are totally different: - -> t = new a.KV({name:'test',env,filter:['bar.>']}); await t.init(); -> await t.get() -{} -> await t.set({"bar.abc":10}) -undefined -> await t.get() -{ 'bar.abc': Uint8Array(2) [ 49, 48 ] } -> await s.get() -{ 'foo.x': 10, 'foo.bar': 20, 'bar.abc': 10 } +> t = await require("@cocalc/backend/nats/sync").kv({name:'test'}) -// The union: -> u = new a.KV({name:'test',env,filter:['bar.>', 'foo.>']}); await u.init(); -> u.get() -{ 'foo.x': 10, 'foo.bar': 20, 'bar.abc': 10 } -> await s.set({'foo.x':999}) -undefined -> u.get() -{ 'foo.x': 999, 'foo.bar': 20, 'bar.abc': 10 } */ import { EventEmitter } from "events"; import { type NatsEnv } from "@cocalc/nats/types"; -import { Kvm } from "@nats-io/kv"; -import { getAllFromKv, matchesPattern } from "@cocalc/nats/util"; -import { isEqual } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { map as awaitMap } from "awaiting"; - -const MAX_PARALLEL = 50; +import { GeneralKV } from "./general-kv"; +import { jsName } from "@cocalc/nats/names"; + +export interface KVOptions { + name: string; + account_id?: string; + project_id?: string; + env: NatsEnv; +} export class KV extends EventEmitter { - public readonly name: string; - private options?; - private filter?: string[]; - private env: NatsEnv; - private kv?; - private watch?; - private all?: { [key: string]: any }; - private revisions?: { [key: string]: number }; - private times?: { [key: string]: Date }; + generalKV?: GeneralKV; + name: string; - constructor({ - name, - env, - filter, - options, - }: { - name: string; - // filter: optionally restrict to subset of named kv store matching these subjects. - // NOTE: any key name that you *set or delete* should match one of these - filter?: string | string[]; - env: NatsEnv; - options?; - }) { + constructor({ name, account_id, project_id, env }: KVOptions) { super(); - this.env = env; + // name of the jetstream key:value store. + const kvname = jsName({ account_id, project_id }); this.name = name; - this.options = options; - this.filter = typeof filter == "string" ? [filter] : filter; + this.generalKV = new GeneralKV({ + name: kvname, + filter: `${name}.>`, + env, + }); + this.init(); + return new Proxy(this, { + deleteProperty(target, prop) { + target.delete(prop); + return true; + }, + set(target, prop, value) { + prop = String(prop); + if (prop == "_eventsCount") { + target[prop] = value; + return true; + } + if (target[prop] != null) { + throw Error(`method name '${prop}' is read only`); + } + target.set(prop, value); + return true; + }, + get(target, prop) { + return target[String(prop)] ?? target.get(String(prop)); + }, + }); } init = reuseInFlight(async () => { - if (this.all != null) { - return; + if (this.generalKV == null) { + throw Error("closed"); } - const kvm = new Kvm(this.env.nc); - this.kv = await kvm.create(this.name, { - compression: true, - ...this.options, - }); - const { all, revisions, times } = await getAllFromKv({ - kv: this.kv, - key: this.filter, - }); - this.revisions = revisions; - this.times = times; - for (const key in all) { - all[key] = this.env.jc.decode(all[key]); - } - this.all = all; - this.emit("connected"); - this.startWatch(); + await this.generalKV.init(); }); - private startWatch = async () => { - // watch for changes - this.watch = await this.kv.watch({ - ignoreDeletes: false, - include: "updates", - key: this.filter, - }); - for await (const x of this.watch) { - const { revision, key, value, sm } = x; - if (this.revisions == null || this.all == null || this.times == null) { - return; - } - this.revisions[key] = revision; - const prev = this.all[key]; - if (value.length == 0) { - // delete - delete this.all[key]; - delete this.times[key]; - } else { - this.all[key] = this.env.jc.decode(value); - this.times[key] = sm.time; - } - this.emit("change", key, this.all[key], prev); - } - }; - close = () => { - this.watch?.stop(); - delete this.all; - delete this.times; - delete this.revisions; + if (this.generalKV == null) { + return; + } + this.generalKV.close(); + delete this.generalKV; this.emit("closed"); this.removeAllListeners(); }; - get = (key?) => { - if (this.all == null) { - throw Error("not initialized"); - } - if (key == undefined) { - return { ...this.all }; - } else { - return this.all?.[key]; + delete = (key) => { + if (this.generalKV == null) { + throw Error("closed"); } + this.generalKV.delete(`${this.name}.${key}`); }; - time = (key?) => { - if (key == null) { - return this.times; - } else { - return this.times?.[key]; - } - }; - - assertValidKey = (key: string) => { - if (!this.isValidKey(key)) { - throw Error( - `delete: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, - ); - } - }; - - isValidKey = (key: string) => { - if (this.filter == null) { - return true; - } - for (const pattern of this.filter) { - if (matchesPattern({ pattern, subject: key })) { - return true; - } - } - return false; - }; - - delete = async (key, revision?) => { - this.assertValidKey(key); - if (this.all == null || this.revisions == null || this.times == null) { - throw Error("not ready"); + get = (key?) => { + if (this.generalKV == null) { + throw Error("closed"); } - if (this.all[key] !== undefined) { - const cur = this.all[key]; - try { - delete this.all[key]; - const newRevision = await this.kv.delete(key, { - previousSeq: revision ?? this.revisions[key], - }); - this.revisions[key] = newRevision; - delete this.times[key]; - } catch (err) { - if (cur === undefined) { - delete this.all[key]; - } else { - this.all[key] = cur; - } - throw err; + if (key == null) { + const obj = this.generalKV.get(); + const x: any = {}; + for (const k in obj) { + x[k.slice(this.name.length + 1)] = obj[k]; } + return x; + } else { + return this.generalKV.get(`${this.name}.${key}`); } }; - // delete everything matching the filter that hasn't been set - // in the given amount of ms. Returns number of deleted records. - // NOTE: This could throw an exception if something that would expire - // were changed right when this is run then it would get expired - // but shouldn't. In that case, run it again. - expire = async (ageMs: number): Promise => { - if (!ageMs) { - throw Error("ageMs must be set"); - } - if (this.times == null || this.all == null) { - throw Error("not initialized"); - } - const cutoff = new Date(Date.now() - ageMs); - // make copy of revisions *before* we start deleting so that - // if a key is changed exactly while deleting we get an error - // and don't accidently delete it! - const revisions = { ...this.revisions }; - const toDelete = Object.keys(this.all).filter( - (key) => this.times?.[key] != null && this.times[key] <= cutoff, - ); - if (toDelete.length > 0) { - await awaitMap(toDelete, MAX_PARALLEL, async (key) => { - await this.delete(key, revisions[key]); - }); - } - return toDelete.length; - }; - - // delete all that we know about - clear = async () => { - if (this.all == null) { - throw Error("not initialized"); - } - await awaitMap(Object.keys(this.all), MAX_PARALLEL, this.delete); - }; - - set = async (...args) => { - if (args.length == 2) { - await this.setOne(args[0], args[1]); - return; + set = async (key: string, value: any) => { + if (this.generalKV == null) { + throw Error("closed"); } - const obj = args[0]; - await awaitMap( - Object.keys(obj), - MAX_PARALLEL, - async (key) => await this.setOne(key, obj[key]), - ); + await this.generalKV.set(`${this.name}.${key}`, value); }; +} - private setOne = async (key, value) => { - if (!this.isValidKey(key)) { - throw Error( - `set: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, - ); - } - if (this.all == null || this.revisions == null) { - throw Error("not ready"); - } - if (isEqual(this.all[key], value)) { - return; - } - if (value === undefined) { - return await this.delete(key); - } - const revision = this.revisions[key]; - const val = this.env.jc.encode(value); - await this.kv.put(key, val, { - previousSeq: revision, - }); - }; +export function userKvKey(options: KVOptions) { + if (!options.name) { + throw Error("name must be specified"); + } + const { env, ...x } = options; + return JSON.stringify(x); } + +const cache: { [key: string]: KV } = {}; +export const kv = reuseInFlight( + async (opts: KVOptions) => { + const key = userKvKey(opts); + if (cache[key] == null) { + const k = new KV(opts); + await k.init(); + k.on("closed", () => delete cache[key]); + cache[key] = k; + } + return cache[key]!; + }, + { + createKey: (args) => userKvKey(args[0]), + }, +); diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 2e07e2f29f..b59c5ff384 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -47,6 +47,9 @@ import { nanos, type Nanos } from "@cocalc/nats/util"; import { delay } from "awaiting"; import { throttle } from "lodash"; import { isNumericString } from "@cocalc/util/misc"; +import { map as awaitMap } from "awaiting"; + +const MAX_PARALLEL = 50; // confirm that ephemeral consumer still exists every 15 seconds: // In case of a long disconnect from the network, this is what @@ -202,6 +205,10 @@ export class Stream extends EventEmitter { return this.messages.length; } + push = async (...args) => { + await awaitMap(args, MAX_PARALLEL, this.publish); + }; + publish = async (mesg: any, subject?: string, options?) => { if (this.js == null) { throw Error("closed"); diff --git a/src/packages/nats/system.ts b/src/packages/nats/system.ts index c10a119d83..73b7b94a6f 100644 --- a/src/packages/nats/system.ts +++ b/src/packages/nats/system.ts @@ -19,9 +19,9 @@ Type ".help" for more information. */ -import { KV } from "@cocalc/nats/sync/kv"; +import { GeneralKV } from "@cocalc/nats/sync/general-kv"; -export class SystemKv extends KV { +export class SystemKv extends GeneralKV { constructor(env) { super({ env, name: "system" }); } From 58d7066c2bd2bede3239967c4b4f8929dc3c69a3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 00:41:06 +0000 Subject: [PATCH 146/281] nats: make it so our streams/kv's can have *arbitrary* names, rather than the very restricted subject segment names --- src/packages/backend/nats/sync.ts | 13 +++++++++---- src/packages/nats/sync/dkv.ts | 16 +++++++++++----- src/packages/nats/sync/dstream.ts | 8 ++++++-- src/packages/nats/sync/kv.ts | 13 ++++++++----- src/packages/nats/sync/stream.ts | 24 ++++++++++++++++-------- 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/src/packages/backend/nats/sync.ts b/src/packages/backend/nats/sync.ts index 8e4c614682..b71efe1ee8 100644 --- a/src/packages/backend/nats/sync.ts +++ b/src/packages/backend/nats/sync.ts @@ -1,9 +1,14 @@ -import { stream as createStream } from "@cocalc/nats/sync/stream"; -import { dstream as createDstream } from "@cocalc/nats/sync/dstream"; -import { kv as createKV } from "@cocalc/nats/sync/kv"; -import { dkv as createDKV } from "@cocalc/nats/sync/dkv"; +import { stream as createStream, type Stream } from "@cocalc/nats/sync/stream"; +import { + dstream as createDstream, + type DStream, +} from "@cocalc/nats/sync/dstream"; +import { kv as createKV, type KV } from "@cocalc/nats/sync/kv"; +import { dkv as createDKV, type DKV } from "@cocalc/nats/sync/dkv"; import { getEnv } from "@cocalc/backend/nats/env"; +export type { Stream, DStream, KV, DKV }; + export async function stream(opts) { return await createStream({ ...opts, env: await getEnv() }); } diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index ae5fbb2bed..1d564da438 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -16,6 +16,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { GeneralDKV, type MergeFunction } from "./general-dkv"; import { userKvKey, type KVOptions } from "./kv"; import { jsName } from "@cocalc/nats/names"; +import { sha1 } from "@cocalc/util/misc"; export interface DKVOptions extends KVOptions { merge: MergeFunction; @@ -24,15 +25,20 @@ export interface DKVOptions extends KVOptions { export class DKV extends EventEmitter { generalDKV?: GeneralDKV; name: string; + private prefix: string; constructor({ name, account_id, project_id, merge, env }: DKVOptions) { super(); + if (env == null) { + throw Error("env must not be null"); + } // name of the jetstream key:value store. const kvname = jsName({ account_id, project_id }); this.name = name; + this.prefix = (env.sha1 ?? sha1)(name); this.generalDKV = new GeneralDKV({ name: kvname, - filter: `${name}.>`, + filter: `${this.prefix}.>`, env, merge, }); @@ -81,7 +87,7 @@ export class DKV extends EventEmitter { if (this.generalDKV == null) { throw Error("closed"); } - this.generalDKV.delete(`${this.name}.${key}`); + this.generalDKV.delete(`${this.prefix}.${key}`); }; get = (key?) => { @@ -92,11 +98,11 @@ export class DKV extends EventEmitter { const obj = this.generalDKV.get(); const x: any = {}; for (const k in obj) { - x[k.slice(this.name.length + 1)] = obj[k]; + x[k.slice(this.prefix.length + 1)] = obj[k]; } return x; } else { - return this.generalDKV.get(`${this.name}.${key}`); + return this.generalDKV.get(`${this.prefix}.${key}`); } }; @@ -104,7 +110,7 @@ export class DKV extends EventEmitter { if (this.generalDKV == null) { throw Error("closed"); } - this.generalDKV.set(`${this.name}.${key}`, value); + this.generalDKV.set(`${this.prefix}.${key}`, value); }; } diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 9c84f9746b..cd1049600b 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -29,10 +29,12 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { delay } from "awaiting"; import { map as awaitMap } from "awaiting"; import { isNumericString } from "@cocalc/util/misc"; +import { sha1 } from "@cocalc/util/misc"; const MAX_PARALLEL = 50; export class DStream extends EventEmitter { + public readonly name: string; private stream?: Stream; private messages: any[]; private raw: any[]; @@ -40,6 +42,7 @@ export class DStream extends EventEmitter { constructor(opts: StreamOptions) { super(); + this.name = opts.name; this.stream = new Stream(opts); this.messages = this.stream.messages; this.raw = this.stream.raw; @@ -175,12 +178,13 @@ export const dstream = reuseInFlight( const { account_id, project_id, name } = options; const jsname = jsName({ account_id, project_id }); const subjects = streamSubject({ account_id, project_id }); - const filter = subjects.replace(">", name); + const filter = subjects.replace(">", (options.env.sha1 ?? sha1)(name)); const key = userStreamOptionsKey(options); if (dstreamCache[key] == null) { const dstream = new DStream({ ...options, - name: jsname, + name, + jsname, subjects, subject: filter, filter, diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index e25fe3b3db..75160b4ede 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -16,6 +16,7 @@ import { type NatsEnv } from "@cocalc/nats/types"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { GeneralKV } from "./general-kv"; import { jsName } from "@cocalc/nats/names"; +import { sha1 } from "@cocalc/util/misc"; export interface KVOptions { name: string; @@ -27,15 +28,17 @@ export interface KVOptions { export class KV extends EventEmitter { generalKV?: GeneralKV; name: string; + private prefix: string; constructor({ name, account_id, project_id, env }: KVOptions) { super(); // name of the jetstream key:value store. const kvname = jsName({ account_id, project_id }); this.name = name; + this.prefix = (env.sha1 ?? sha1)(name); this.generalKV = new GeneralKV({ name: kvname, - filter: `${name}.>`, + filter: `${this.prefix}.>`, env, }); this.init(); @@ -83,7 +86,7 @@ export class KV extends EventEmitter { if (this.generalKV == null) { throw Error("closed"); } - this.generalKV.delete(`${this.name}.${key}`); + this.generalKV.delete(`${this.prefix}.${key}`); }; get = (key?) => { @@ -94,11 +97,11 @@ export class KV extends EventEmitter { const obj = this.generalKV.get(); const x: any = {}; for (const k in obj) { - x[k.slice(this.name.length + 1)] = obj[k]; + x[k.slice(this.prefix.length + 1)] = obj[k]; } return x; } else { - return this.generalKV.get(`${this.name}.${key}`); + return this.generalKV.get(`${this.prefix}.${key}`); } }; @@ -106,7 +109,7 @@ export class KV extends EventEmitter { if (this.generalKV == null) { throw Error("closed"); } - await this.generalKV.set(`${this.name}.${key}`, value); + await this.generalKV.set(`${this.prefix}.${key}`, value); }; } diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index b59c5ff384..b8c0fb5832 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -48,6 +48,7 @@ import { delay } from "awaiting"; import { throttle } from "lodash"; import { isNumericString } from "@cocalc/util/misc"; import { map as awaitMap } from "awaiting"; +import { sha1 } from "@cocalc/util/misc"; const MAX_PARALLEL = 50; @@ -88,7 +89,10 @@ interface FilteredStreamLimitOptions { } export interface StreamOptions { + // what it's called by us name: string; + // actually name of the jetstream in NATS + jsname: string; // subject = default subject used for publishing; defaults to filter if filter doesn't have any wildcard subjects: string | string[]; subject?: string; @@ -102,6 +106,7 @@ export interface StreamOptions { export class Stream extends EventEmitter { public readonly name: string; + public readonly jsname: string; private natsStreamOptions?; private limits: FilteredStreamLimitOptions; private subjects: string | string[]; @@ -119,6 +124,7 @@ export class Stream extends EventEmitter { constructor({ name, + jsname, env, subject, subjects, @@ -132,6 +138,7 @@ export class Stream extends EventEmitter { // create a jetstream client so we can publish to the stream this.js = jetstream(env.nc); this.name = name; + this.jsname = jsname; this.natsStreamOptions = natsStreamOptions; if ( subject == null && @@ -178,12 +185,12 @@ export class Stream extends EventEmitter { }; try { this.stream = await this.jsm.streams.add({ - name: this.name, + name: this.jsname, ...options, }); } catch (err) { // probably already exists, so try to modify to have the requested properties. - this.stream = await this.jsm.streams.update(this.name, options); + this.stream = await this.jsm.streams.update(this.jsname, options); } this.startFetch(); }); @@ -251,11 +258,11 @@ export class Stream extends EventEmitter { } else { startOptions = {}; } - const { name } = await jsm.consumers.add(this.name, { + const { name } = await jsm.consumers.add(this.jsname, { ...options, ...startOptions, }); - return await js.consumers.get(this.name, name); + return await js.consumers.get(this.jsname, name); }; private startFetch = async (options?) => { @@ -363,12 +370,12 @@ export class Stream extends EventEmitter { index = this.raw.length - 1; // everything // console.log("purge everything"); - await this.jsm.streams.purge(this.name, { + await this.jsm.streams.purge(this.jsname, { filter: this.filter, }); } else { const { seq } = this.raw[index + 1]; - await this.jsm.streams.purge(this.name, { + await this.jsm.streams.purge(this.jsname, { filter: this.filter, seq, }); @@ -511,12 +518,13 @@ export const stream = reuseInFlight( const { account_id, project_id, name } = options; const jsname = jsName({ account_id, project_id }); const subjects = streamSubject({ account_id, project_id }); - const filter = subjects.replace(">", name); + const filter = subjects.replace(">", (options.env.sha1 ?? sha1)(name)); const key = userStreamOptionsKey(options); if (streamCache[key] == null) { const stream = new Stream({ ...options, - name: jsname, + name, + jsname, subjects, subject: filter, filter, From fa1aff296f83072f15ea71cd063c9c6753e2317e Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 01:01:06 +0000 Subject: [PATCH 147/281] nats kv: make it possible to use *arbitrary strings* for keys - we use the sha1 hash of the key in nats and keep track of original key as part of the value - pros and cons: - pro: a lot of probably bugs and subtle ugly code with dangerous conventions doesn't have to get written, and it would basically do the same thing - con: harder to observer what is going on (e.g., impossible with nats cli)... but we can certainly build tooling --- src/packages/nats/sync/dkv.ts | 13 ++++++++----- src/packages/nats/sync/kv.ts | 26 ++++++++++++++++++++------ 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index 1d564da438..bbfd0a21f9 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -26,6 +26,7 @@ export class DKV extends EventEmitter { generalDKV?: GeneralDKV; name: string; private prefix: string; + private sha1; constructor({ name, account_id, project_id, merge, env }: DKVOptions) { super(); @@ -35,7 +36,8 @@ export class DKV extends EventEmitter { // name of the jetstream key:value store. const kvname = jsName({ account_id, project_id }); this.name = name; - this.prefix = (env.sha1 ?? sha1)(name); + this.sha1 = env.sha1 ?? sha1; + this.prefix = this.sha1(name); this.generalDKV = new GeneralDKV({ name: kvname, filter: `${this.prefix}.>`, @@ -87,7 +89,7 @@ export class DKV extends EventEmitter { if (this.generalDKV == null) { throw Error("closed"); } - this.generalDKV.delete(`${this.prefix}.${key}`); + this.generalDKV.delete(`${this.prefix}.${this.sha1(key)}`); }; get = (key?) => { @@ -98,11 +100,12 @@ export class DKV extends EventEmitter { const obj = this.generalDKV.get(); const x: any = {}; for (const k in obj) { - x[k.slice(this.prefix.length + 1)] = obj[k]; + const { key, value } = obj[k]; + x[key] = value; } return x; } else { - return this.generalDKV.get(`${this.prefix}.${key}`); + return this.generalDKV.get(`${this.prefix}.${this.sha1(key)}`)?.value; } }; @@ -110,7 +113,7 @@ export class DKV extends EventEmitter { if (this.generalDKV == null) { throw Error("closed"); } - this.generalDKV.set(`${this.prefix}.${key}`, value); + this.generalDKV.set(`${this.prefix}.${this.sha1(key)}`, { key, value }); }; } diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index 75160b4ede..82d07cced3 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -29,13 +29,15 @@ export class KV extends EventEmitter { generalKV?: GeneralKV; name: string; private prefix: string; + private sha1; constructor({ name, account_id, project_id, env }: KVOptions) { super(); // name of the jetstream key:value store. const kvname = jsName({ account_id, project_id }); this.name = name; - this.prefix = (env.sha1 ?? sha1)(name); + this.sha1 = env.sha1 ?? sha1; + this.prefix = this.sha1(name); this.generalKV = new GeneralKV({ name: kvname, filter: `${this.prefix}.>`, @@ -82,11 +84,19 @@ export class KV extends EventEmitter { this.removeAllListeners(); }; - delete = (key) => { + delete = async (key) => { if (this.generalKV == null) { throw Error("closed"); } - this.generalKV.delete(`${this.prefix}.${key}`); + await this.generalKV.delete(`${this.prefix}.${this.sha1(key)}`); + }; + + // delete everything + clear = async () => { + if (this.generalKV == null) { + throw Error("closed"); + } + await this.generalKV.clear(); }; get = (key?) => { @@ -97,11 +107,12 @@ export class KV extends EventEmitter { const obj = this.generalKV.get(); const x: any = {}; for (const k in obj) { - x[k.slice(this.prefix.length + 1)] = obj[k]; + const { key, value } = obj[k]; + x[key] = value; } return x; } else { - return this.generalKV.get(`${this.prefix}.${key}`); + return this.generalKV.get(`${this.prefix}.${this.sha1(key)}`)?.value; } }; @@ -109,7 +120,10 @@ export class KV extends EventEmitter { if (this.generalKV == null) { throw Error("closed"); } - await this.generalKV.set(`${this.prefix}.${key}`, value); + await this.generalKV.set(`${this.prefix}.${this.sha1(key)}`, { + key, + value, + }); }; } From 34f1457f83d34f5b985d82c94d920736a0c383e6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 02:01:20 +0000 Subject: [PATCH 148/281] nats: switching to new stream for terminal -- work in progress that doesn't work yet --- .../terminal-editor/connected-terminal.ts | 4 +- .../nats-terminal-connection.ts | 78 ++++++------------- src/packages/nats/sync/stream.ts | 25 ++++-- src/packages/project/nats/sync.ts | 26 +++++++ src/packages/project/nats/terminal.ts | 53 +++---------- 5 files changed, 82 insertions(+), 104 deletions(-) create mode 100644 src/packages/project/nats/sync.ts diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index d5f72f28ed..2e194c9a1a 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -221,7 +221,7 @@ export class Terminal { } } - close(): void { + close = (): void => { this.assert_not_closed(); this.set_connection_status("disconnected"); this.state = "closed"; @@ -233,7 +233,7 @@ export class Terminal { } close(this); this.state = "closed"; - } + }; private disconnect(): void { if (this.conn === undefined) { diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 4e2b7299aa..c5ad3263ce 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -4,7 +4,8 @@ import { JSONCodec } from "nats.ws"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { uuid } from "@cocalc/util/misc"; import { delay } from "awaiting"; -import { projectStreamName, projectSubject } from "@cocalc/nats/names"; +import { type DStream } from "@cocalc/nats/sync/dstream"; +import { projectSubject } from "@cocalc/nats/names"; const jc = JSONCodec(); const client = uuid(); @@ -13,10 +14,9 @@ export class NatsTerminalConnection extends EventEmitter { private project_id: string; //private compute_server_id: number; private path: string; - private subject: string; private cmd_subject: string; private state: null | "running" | "init" | "closed"; - private consumer?; + private stream?: DStream; // keep = optional number of messages to retain between clients/sessions/view, i.e., // "amount of history". This is global to all terminals in the project. private keep?: number; @@ -51,12 +51,6 @@ export class NatsTerminalConnection extends EventEmitter { this.openPaths = openPaths; this.closePaths = closePaths; this.project = webapp_client.nats_client.projectApi({ project_id }); - this.subject = projectSubject({ - project_id, - compute_server_id, - service: "terminal", - path, - }); this.cmd_subject = projectSubject({ project_id, compute_server_id, @@ -109,7 +103,9 @@ export class NatsTerminalConnection extends EventEmitter { }; end = () => { - // todo + this.stream?.close(); + delete this.stream; + // todo -- anything else? this.state = "closed"; }; @@ -118,42 +114,23 @@ export class NatsTerminalConnection extends EventEmitter { await this.project.terminal.create({ path: this.path }); }); - private getConsumer = async () => { + private getStream = async () => { // TODO: idempotent, but move to project const { nats_client } = webapp_client; - const streamName = projectStreamName({ + return await nats_client.dstream({ + name: `terminal-${this.path}`, project_id: this.project_id, - service: "terminal", - }); - const nc = await nats_client.getConnection(); - const js = nats_client.jetstream.jetstream(nc); - // consumer doesn't exist, so setup everything. - const jsm = await nats_client.jetstream.jetstreamManager(nc); - // making an ephemeral consumer for just one subject (e.g., this terminal frame) - const { name } = await jsm.consumers.add(streamName, { - filter_subject: this.subject, }); - return await js.consumers.get(streamName, name); }; init = async () => { this.state = "init"; await this.start(); - this.consumer = await this.getConsumer(); + this.stream = await this.getStream(); this.consumeDataStream(); this.subscribeToCommands(); }; - private handle = (mesg) => { - if (this.state == "closed") { - return true; - } - const x = jc.decode(mesg.data) as any; - if (x?.data != null) { - this.emit("data", x?.data); - } - }; - private subscribeToCommands = async () => { const nc = await webapp_client.nats_client.getConnection(); const sub = nc.subscribe(this.cmd_subject); @@ -184,33 +161,22 @@ export class NatsTerminalConnection extends EventEmitter { } }; - private consumeDataStream = async () => { - if (this.consumer == null) { + private handleStreamMessage = (mesg) => { + const data = mesg?.data; + if (data) { + this.emit("data", data); + } + }; + + private consumeDataStream = () => { + if (this.stream == null) { return; } - const messages = await this.consumer.fetch({ - max_messages: 100000, // should only be a few hundred in practice - expires: 1000, - }); - for await (const mesg of messages) { - if (this.handle(mesg)) { - return; - } - if (mesg.info.pending == 0) { - // no further messages pending, so switch to consuming below - // TODO: I don't know if there is some chance to miss a message? - // This is a *terminal* so purely visual so not too critical. - break; - } + for (const mesg of this.stream.get()) { + this.handleStreamMessage(mesg); } - this.setReady(); - - for await (const mesg of await this.consumer.consume()) { - if (this.handle(mesg)) { - return; - } - } + this.stream.on("change", this.handleStreamMessage); }; private setReady = async () => { diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index b8c0fb5832..0437c2b5dd 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -192,7 +192,12 @@ export class Stream extends EventEmitter { // probably already exists, so try to modify to have the requested properties. this.stream = await this.jsm.streams.update(this.jsname, options); } - this.startFetch(); + const consumer = await this.fetchInitialData(); + if (this.stream == null) { + // closed *during* initial load + return; + } + this.watchForNewData(consumer); }); get = (n?) => { @@ -265,7 +270,7 @@ export class Stream extends EventEmitter { return await js.consumers.get(this.jsname, name); }; - private startFetch = async (options?) => { + private fetchInitialData = async (options?) => { const consumer = await this.getConsumer(options); // This goes in two stages: // STAGE 1: Get what is in the stream now. @@ -284,6 +289,10 @@ export class Stream extends EventEmitter { break; } } + return consumer; + }; + + private watchForNewData = async (consumer) => { if (this.stream == null) { // closed *during* initial load return; @@ -319,12 +328,18 @@ export class Stream extends EventEmitter { this.watch.stop(); // stop current watch // make new one: const start_seq = this.raw[this.raw.length - 1]?.seq + 1; - this.startFetch({ start_seq }); - return; // because startFetch creates a new consumer monitor loop + const consumer = await this.fetchInitialData({ start_seq }); + if (this.stream == null) { + // closed + return; + } + this.watchForNewData(consumer); + + return; // because watchForNewData creates a new consumer monitor loop } } + await delay(CONSUMER_MONITOR_INTERVAL); } - await delay(CONSUMER_MONITOR_INTERVAL); }; private decode = (raw) => { diff --git a/src/packages/project/nats/sync.ts b/src/packages/project/nats/sync.ts new file mode 100644 index 0000000000..4c776ee80f --- /dev/null +++ b/src/packages/project/nats/sync.ts @@ -0,0 +1,26 @@ +import { stream as createStream, type Stream } from "@cocalc/nats/sync/stream"; +import { + dstream as createDstream, + type DStream, +} from "@cocalc/nats/sync/dstream"; +import { kv as createKV, type KV } from "@cocalc/nats/sync/kv"; +import { dkv as createDKV, type DKV } from "@cocalc/nats/sync/dkv"; +import { getEnv } from "./env"; + +export type { Stream, DStream, KV, DKV }; + +export async function stream(opts) { + return await createStream({ ...opts, env: await getEnv() }); +} + +export async function dstream(opts) { + return await createDstream({ ...opts, env: await getEnv() }); +} + +export async function kv(opts) { + return await createKV({ ...opts, env: await getEnv() }); +} + +export async function dkv(opts) { + return await createDKV({ ...opts, env: await getEnv() }); +} diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index db52d3332a..2aeb00d369 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -11,17 +11,15 @@ import { console_init_filename, len } from "@cocalc/util/misc"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { JSONCodec } from "nats"; -import { jetstreamManager } from "@nats-io/jetstream"; import { getLogger } from "@cocalc/project/logger"; import { readlink, realpath } from "node:fs/promises"; +import { dstream, type DStream } from "@cocalc/project/nats/sync"; +import { project_id } from "@cocalc/project/data"; +import { getSubject } from "./names"; import getConnection from "./connection"; -import { getSubject, getStreamName } from "./names"; const logger = getLogger("server:nats:terminal"); -const DEFAULT_KEEP = 300; -const MIN_KEEP = 5; -const MAX_KEEP = 2000; const EXIT_MESSAGE = "\r\n\r\n[Process completed - press any key]\r\n\r\n"; const DEFAULT_COMMAND = "/bin/bash"; const INFINITY = 999999; @@ -86,7 +84,6 @@ export async function terminalCommand({ path, cmd, ...args }) { } class Session { - private nc; private path: string; private options; private pty?; @@ -95,21 +92,17 @@ class Session { public subject: string; private cmd_subject: string; private state: "running" | "off" = "off"; + private stream: DStream; private streamName: string; - private keep: number; + private nc; constructor({ path, options, nc }) { logger.debug("create session ", { path, options }); - this.nc = nc; this.path = path; this.options = options; - this.keep = Math.max( - MIN_KEEP, - Math.min(this.options.keep ?? DEFAULT_KEEP, MAX_KEEP), - ); - this.subject = getSubject({ service: "terminal", path }); this.cmd_subject = getSubject({ service: "terminal-cmd", path }); - this.streamName = getStreamName({ service: "terminal" }); + this.streamName = `terminal-${path}`; + this.nc = nc; } write = async (data) => { @@ -121,6 +114,7 @@ class Session { restart = async () => { this.pty?.destroy(); + this.stream?.close(); delete this.pty; await this.init(); }; @@ -147,24 +141,7 @@ class Session { }; createStream = async () => { - // idempotent so don't have to check if there is already a stream - const nc = this.nc; - const jsm = await jetstreamManager(nc); - try { - await jsm.streams.add({ - name: this.streamName, - subjects: [getSubject({ service: "terminal" }) + ".>"], - compression: "s2", - max_msgs_per_subject: this.keep, - }); - } catch (_err) { - // probably already exists - await jsm.streams.update(this.streamName, { - subjects: [getSubject({ service: "terminal" }) + ".>"], - compression: "s2" as any, - max_msgs_per_subject: this.keep, - }); - } + this.stream = await dstream({ name: this.streamName, project_id }); }; init = async () => { @@ -194,19 +171,15 @@ class Session { await this.createStream(); this.pty.onData((data) => { this.handleBackendMessages(data); - this.publish({ data }); + this.stream.publish({ data }); }); this.pty.onExit((status) => { - this.publish({ data: EXIT_MESSAGE }); - this.publish({ ...status, exit: true }); + this.stream.publish({ data: EXIT_MESSAGE }); + this.stream.publish({ ...status, exit: true }); this.state = "off"; }); }; - private publish = (mesg) => { - this.nc.publish(this.subject, jc.encode(mesg)); - }; - private publishCommand = (mesg) => { this.nc.publish(this.cmd_subject, jc.encode(mesg)); }; @@ -319,13 +292,11 @@ class Session { if (i == -1) { // continue to wait... unless too long if (this.backendMessagesBuffer.length > 10000) { - console.log("huge reset"); this.resetBackendMessagesBuffer(); } return; } const s = this.backendMessagesBuffer.slice(5, i); - console.log("endup up with ", { s }); this.resetBackendMessagesBuffer(); logger.debug( `handle_backend_message: parsing JSON payload ${JSON.stringify(s)}`, From 1f2a47595c0c86cf4aafeabbc6ab526308bfdf48 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 04:13:05 +0000 Subject: [PATCH 149/281] fix a circular reference issue --- src/packages/nats/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/packages/nats/tsconfig.json b/src/packages/nats/tsconfig.json index a0c8debf1a..687201523d 100644 --- a/src/packages/nats/tsconfig.json +++ b/src/packages/nats/tsconfig.json @@ -5,5 +5,6 @@ "outDir": "dist" }, "exclude": ["node_modules", "dist", "test"], - "references": [{ "path": "../util" }, { "path": "../comm" }] + "references_comment": "Do not define path:../comm because that causes a circular references.", + "references": [{ "path": "../util" }] } From e5833500641edd8763d06cd1cc0dd83585f9013b Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 04:37:27 +0000 Subject: [PATCH 150/281] nats: new streams were hanging on startup --- src/packages/backend/nats/sync.ts | 8 ++++---- src/packages/nats/sync/stream.ts | 3 +++ src/packages/project/nats/api/index.ts | 13 ++++++++----- src/packages/project/nats/sync.ts | 9 +++++---- src/packages/project/nats/terminal.ts | 7 ++++--- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/packages/backend/nats/sync.ts b/src/packages/backend/nats/sync.ts index b71efe1ee8..c59a632752 100644 --- a/src/packages/backend/nats/sync.ts +++ b/src/packages/backend/nats/sync.ts @@ -10,17 +10,17 @@ import { getEnv } from "@cocalc/backend/nats/env"; export type { Stream, DStream, KV, DKV }; export async function stream(opts) { - return await createStream({ ...opts, env: await getEnv() }); + return await createStream({ env: await getEnv(), ...opts }); } export async function dstream(opts) { - return await createDstream({ ...opts, env: await getEnv() }); + return await createDstream({ env: await getEnv(), ...opts }); } export async function kv(opts) { - return await createKV({ ...opts, env: await getEnv() }); + return await createKV({ env: await getEnv(), ...opts }); } export async function dkv(opts) { - return await createDKV({ ...opts, env: await getEnv() }); + return await createDKV({ env: await getEnv(), ...opts }); } diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 0437c2b5dd..1f50ca611f 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -277,6 +277,9 @@ export class Stream extends EventEmitter { // First we get info so we know how many messages // are already in the stream: const info = await consumer.info(); + if (info.num_pending == 0) { + return consumer; + } const fetch = await consumer.fetch(); this.watch = fetch; let i = 0; diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index 39de6befac..4f5636bf23 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -1,17 +1,20 @@ /* How to do development (so in a dev project doing cc-in-cc dev). -0. From the browser, terminate this api server running in the project already, if any +0. From the browser, terminate this api server running in the project: - await cc.client.nats_client.projectApi({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'}).system.terminate({service:'api'}) + > await cc.client.nats_client.projectApi({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'}).system.terminate({service:'api'}) + + {status: 'terminated', service: 'api'} 1. Open a terminal in the project itself, which sets up the required environment variables, e.g., - COCALC_NATS_JWT -- this has the valid JWT issued to grant the project rights to use nats - COCALC_PROJECT_ID -You can type the following into the miniterminal in a project and copy the output into a terminal here to -setup the same environment and make starting this server act like this part of a project. +You can type the following into the miniterminal in a project and copy the output into +a terminal here to setup the same environment and make starting this server act like +this part of a project. export | grep -E "COCALC|HOME" @@ -21,7 +24,7 @@ setup the same environment and make starting this server act like this part of a or just run node and paste - require("@cocalc/project/nats/api").init() + require("@cocalc/project/nats/api/index").init() if you want to easily be able to grab some state, e.g., global.x = {...} in some code. diff --git a/src/packages/project/nats/sync.ts b/src/packages/project/nats/sync.ts index 4c776ee80f..044b10e2a0 100644 --- a/src/packages/project/nats/sync.ts +++ b/src/packages/project/nats/sync.ts @@ -6,21 +6,22 @@ import { import { kv as createKV, type KV } from "@cocalc/nats/sync/kv"; import { dkv as createDKV, type DKV } from "@cocalc/nats/sync/dkv"; import { getEnv } from "./env"; +import { project_id } from "@cocalc/project/data"; export type { Stream, DStream, KV, DKV }; export async function stream(opts) { - return await createStream({ ...opts, env: await getEnv() }); + return await createStream({ project_id, env: await getEnv(), ...opts }); } export async function dstream(opts) { - return await createDstream({ ...opts, env: await getEnv() }); + return await createDstream({ project_id, env: await getEnv(), ...opts }); } export async function kv(opts) { - return await createKV({ ...opts, env: await getEnv() }); + return await createKV({ project_id, env: await getEnv(), ...opts }); } export async function dkv(opts) { - return await createDKV({ ...opts, env: await getEnv() }); + return await createDKV({ project_id, env: await getEnv(), ...opts }); } diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index 2aeb00d369..b1b37dde97 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -14,7 +14,6 @@ import { JSONCodec } from "nats"; import { getLogger } from "@cocalc/project/logger"; import { readlink, realpath } from "node:fs/promises"; import { dstream, type DStream } from "@cocalc/project/nats/sync"; -import { project_id } from "@cocalc/project/data"; import { getSubject } from "./names"; import getConnection from "./connection"; @@ -141,7 +140,7 @@ class Session { }; createStream = async () => { - this.stream = await dstream({ name: this.streamName, project_id }); + this.stream = await dstream({ name: this.streamName }); }; init = async () => { @@ -160,7 +159,7 @@ class Session { args.push(path_split(initFilename).tail); } const cwd = getCWD(head, this.options.cwd); - logger.debug("creating pty with size", this.size); + logger.debug("creating pty"); this.pty = spawn(command, args, { cwd, env, @@ -168,7 +167,9 @@ class Session { cols: this.size?.cols, }); this.state = "running"; + logger.debug("creating stream"); await this.createStream(); + logger.debug("connect stream to pty"); this.pty.onData((data) => { this.handleBackendMessages(data); this.stream.publish({ data }); From f05877915261999e1fff025b214fe3717543f077 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 04:47:10 +0000 Subject: [PATCH 151/281] nats terminal: limit history size (in bytes) --- src/packages/project/nats/terminal.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index b1b37dde97..f35760beca 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -23,6 +23,8 @@ const EXIT_MESSAGE = "\r\n\r\n[Process completed - press any key]\r\n\r\n"; const DEFAULT_COMMAND = "/bin/bash"; const INFINITY = 999999; +const HISTORY_LIMIT_BYTES = 20000; + const jc = JSONCodec(); const sessions: { [name: string]: Session } = {}; @@ -140,7 +142,10 @@ class Session { }; createStream = async () => { - this.stream = await dstream({ name: this.streamName }); + this.stream = await dstream({ + name: this.streamName, + limits: { max_bytes: HISTORY_LIMIT_BYTES }, + }); }; init = async () => { From e76dfcaad09d57bdee178658d6238a046d5df73f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 12:27:13 +0000 Subject: [PATCH 152/281] nats auth: no longer use -terminal stream --- src/packages/server/nats/auth.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 9ef6cee391..46957d0d5d 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -19,7 +19,7 @@ DOCS: USAGE: a = require('@cocalc/server/nats/auth'); await a.configureNatsUser({account_id:'6aae57c6-08f1-4bb5-848b-3ceb53e61ede'}) -await a.configureNatsUser({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}) +await a.configureNatsUser({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'}) */ import getPool from "@cocalc/database/pool"; @@ -351,7 +351,7 @@ function projectSubjects(project_id: string) { pub.add(`$JS.*.*.*.KV_project-${project_id}`); pub.add(`$JS.*.*.*.KV_project-${project_id}.>`); - for (const name of ["", "-patches", "-terminal"]) { + for (const name of ["", "-patches"]) { pub.add(`$JS.*.*.*.project-${project_id}${name}`); pub.add(`$JS.*.*.*.project-${project_id}${name}.>`); pub.add(`$JS.*.*.*.*.project-${project_id}${name}.>`); From 21636fd36f812b28d67d15f806f94f6501a5bd52 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 13:50:04 +0000 Subject: [PATCH 153/281] nats: implements limits for kv --- src/packages/nats/sync/general-kv.ts | 198 ++++++++++++++++++++++++++- src/packages/nats/sync/kv.ts | 6 +- src/packages/nats/sync/open-files.ts | 1 + src/packages/nats/sync/stream.ts | 5 +- src/packages/nats/util.ts | 1 - 5 files changed, 199 insertions(+), 12 deletions(-) diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index 9f0d29ff9a..29e7e147c2 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -95,13 +95,43 @@ undefined import { EventEmitter } from "events"; import { type NatsEnv } from "@cocalc/nats/types"; import { Kvm } from "@nats-io/kv"; -import { getAllFromKv, matchesPattern } from "@cocalc/nats/util"; +import { + getAllFromKv, + matchesPattern, + millis, + type Nanos, +} from "@cocalc/nats/util"; import { isEqual } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { map as awaitMap } from "awaiting"; +import { throttle } from "lodash"; const MAX_PARALLEL = 50; +// Note that the limit options are named in exactly the same was as for streams, +// which is convenient for consistency. This is not consistent with NATS's +// own KV store limit naming. +const ENFORCE_LIMITS_THROTTLE_MS = 3000; +export interface KVLimits { + // How many keys may be in the KV store. Oldest keys will be removed + // if the key-value store exceeds this size. -1 for unlimited. + max_msgs: number; + + // Maximum age of any key, expressed in nanoseconds. 0 for unlimited. + // Use 'import {nanos} from "@cocalc/nats/util"' then "nanos(milliseconds)" + // to give input in milliseconds. + max_age: Nanos; // nanoseconds! + + // The maximum number of bytes to store in this KV, which means + // the total of the bytes used to store everything. Since we store + // the key with each value (to have arbitrary keys), this includes + // the size of the keys. + max_bytes: number; + + // The maximum size of any single value, including the key. + max_msg_size: number; +} + export class GeneralKV extends EventEmitter { public readonly name: string; private options?; @@ -112,12 +142,15 @@ export class GeneralKV extends EventEmitter { private all?: { [key: string]: any }; private revisions?: { [key: string]: number }; private times?: { [key: string]: Date }; + private sizes?: { [key: string]: number }; + private limits: KVLimits; constructor({ name, env, filter, options, + limits, }: { name: string; // filter: optionally restrict to subset of named kv store matching these subjects. @@ -125,8 +158,17 @@ export class GeneralKV extends EventEmitter { filter?: string | string[]; env: NatsEnv; options?; + limits?: KVLimits; }) { super(); + this.limits = { + max_msgs: -1, + max_age: 0, + max_bytes: -1, + max_msg_size: -1, + ...limits, + }; + this.env = env; this.name = name; this.options = options; @@ -148,7 +190,9 @@ export class GeneralKV extends EventEmitter { }); this.revisions = revisions; this.times = times; + this.sizes = {}; for (const key in all) { + this.sizes[key] = all[key].length; all[key] = this.env.jc.decode(all[key]); } this.all = all; @@ -165,7 +209,12 @@ export class GeneralKV extends EventEmitter { }); for await (const x of this.watch) { const { revision, key, value, sm } = x; - if (this.revisions == null || this.all == null || this.times == null) { + if ( + this.revisions == null || + this.all == null || + this.times == null || + this.sizes == null + ) { return; } this.revisions[key] = revision; @@ -174,11 +223,14 @@ export class GeneralKV extends EventEmitter { // delete delete this.all[key]; delete this.times[key]; + delete this.sizes[key]; } else { this.all[key] = this.env.jc.decode(value); this.times[key] = sm.time; + this.sizes[key] = value.length; } this.emit("change", key, this.all[key], prev); + this.enforceLimits(); } }; @@ -187,6 +239,7 @@ export class GeneralKV extends EventEmitter { delete this.all; delete this.times; delete this.revisions; + delete this.sizes; this.emit("closed"); this.removeAllListeners(); }; @@ -232,7 +285,12 @@ export class GeneralKV extends EventEmitter { delete = async (key, revision?) => { this.assertValidKey(key); - if (this.all == null || this.revisions == null || this.times == null) { + if ( + this.all == null || + this.revisions == null || + this.times == null || + this.sizes == null + ) { throw Error("not ready"); } if (this.all[key] !== undefined) { @@ -244,6 +302,7 @@ export class GeneralKV extends EventEmitter { }); this.revisions[key] = newRevision; delete this.times[key]; + delete this.sizes[key]; } catch (err) { if (cur === undefined) { delete this.all[key]; @@ -260,14 +319,28 @@ export class GeneralKV extends EventEmitter { // NOTE: This could throw an exception if something that would expire // were changed right when this is run then it would get expired // but shouldn't. In that case, run it again. - expire = async (ageMs: number): Promise => { - if (!ageMs) { - throw Error("ageMs must be set"); + expire = async ({ + cutoff, + ageMs, + }: { + cutoff?: Date; + ageMs?: number; + }): Promise => { + if (!ageMs && !cutoff) { + throw Error("one of ageMs or cutoff must be set"); + } + if (ageMs && cutoff) { + throw Error("exactly one of ageMs or cutoff must be set"); } if (this.times == null || this.all == null) { throw Error("not initialized"); } - const cutoff = new Date(Date.now() - ageMs); + if (ageMs && !cutoff) { + cutoff = new Date(Date.now() - ageMs); + } + if (cutoff == null) { + throw Error("impossible"); + } // make copy of revisions *before* we start deleting so that // if a key is changed exactly while deleting we get an error // and don't accidently delete it! @@ -321,8 +394,119 @@ export class GeneralKV extends EventEmitter { } const revision = this.revisions[key]; const val = this.env.jc.encode(value); + if ( + this.limits.max_msg_size > -1 && + val.length > this.limits.max_msg_size + ) { + throw Error( + `message key:value size (=${val.length}) exceeds max_msg_size=${this.limits.max_msg_size} bytes`, + ); + } await this.kv.put(key, val, { previousSeq: revision, }); }; + + // ensure any limits are satisfied, always by deleting old keys + private enforceLimits = throttle( + reuseInFlight(async () => { + if (this.all == null || this.times == null || this.sizes == null) { + return; + } + const { max_msgs, max_age, max_bytes } = this.limits; + let times: { time: Date; key: string }[] | null = null; + const getTimes = (): { time: Date; key: string }[] => { + if (times == null) { + // this is potentially a little worrisome regarding performance, but + // it has to be done, or we have to do something elsewhere to maintain + // this info. The intention with these kv's is they are small and all + // in memory. + const v: { time: Date; key: string }[] = []; + for (const key in this.times) { + v.push({ time: this.times[key], key }); + } + v.sort((a, b) => (a.time < b.time ? -1 : a.time > b.time ? 1 : 0)); + times = v; + } + return times!; + }; + + // we check with each defined limit if some old messages + // should be dropped, and if so move limit forward. If + // it is above -1 at the end, we do the drop. + let index = -1; + const setIndex = (i, _limit) => { + // console.log("setIndex", { i, _limit }); + index = Math.max(i, index); + }; + + //max_msgs = max number of keys + const v = Object.keys(this.all); + if (max_msgs > -1 && v.length > max_msgs) { + // ensure there are at most this.limits.max_msgs messages + // by deleting the oldest ones up to a specified point. + const i = v.length - max_msgs; + if (i > 0) { + setIndex(i - 1, "max_msgs"); + } + } + + // max_age + if (max_age > 0) { + const times = getTimes(); + if (times.length > 1) { + // expire messages older than max_age nanoseconds + // to avoid potential clock skew, we define *now* as the time of the most + // recent message. For us, this should be fine, since we only impose limits + // when writing new messages, and none of these limits are guaranteed. + const now = times[times.length - 1].time.valueOf(); + const cutoff = new Date(now - millis(max_age)); + for (let i = times.length - 2; i >= 0; i--) { + if (times[i].time < cutoff) { + // it just went over the limit. Everything before + // and including the i-th message should be deleted. + setIndex(i, "max_age"); + break; + } + } + } + } + + // max_bytes + if (max_bytes >= 0) { + let t = 0; + const times = getTimes(); + for (let i = times.length - 1; i >= 0; i--) { + t += this.sizes[times[i].key]; + if (t > max_bytes) { + // it just went over the limit. Everything before + // and including the i-th message must be deleted. + setIndex(i, "max_bytes"); + break; + } + } + } + + if (index > -1 && this.times != null) { + try { + // console.log("enforceLimits: deleting ", { index }); + const times = getTimes(); + const toDelete = times.slice(0, index + 1).map(({ key }) => key); + if (toDelete.length > 0) { + // console.log("enforceLImits: deleting ", toDelete.length, " keys"); + const revisions = { ...this.revisions }; + await awaitMap(toDelete, MAX_PARALLEL, async (key) => { + await this.delete(key, revisions[key]); + }); + } + } catch (err) { + if (err.code != "TIMEOUT") { + console.log(`WARNING: expiring old messages - ${err}`); + } + } + } + }), + ENFORCE_LIMITS_THROTTLE_MS, + { leading: true, trailing: true }, + ); } diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index 82d07cced3..22ede8d5fe 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -14,7 +14,7 @@ Type ".help" for more information. import { EventEmitter } from "events"; import { type NatsEnv } from "@cocalc/nats/types"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { GeneralKV } from "./general-kv"; +import { GeneralKV, type KVLimits } from "./general-kv"; import { jsName } from "@cocalc/nats/names"; import { sha1 } from "@cocalc/util/misc"; @@ -23,6 +23,7 @@ export interface KVOptions { account_id?: string; project_id?: string; env: NatsEnv; + limits?: KVLimits; } export class KV extends EventEmitter { @@ -31,7 +32,7 @@ export class KV extends EventEmitter { private prefix: string; private sha1; - constructor({ name, account_id, project_id, env }: KVOptions) { + constructor({ name, account_id, project_id, env, limits }: KVOptions) { super(); // name of the jetstream key:value store. const kvname = jsName({ account_id, project_id }); @@ -42,6 +43,7 @@ export class KV extends EventEmitter { name: kvname, filter: `${this.prefix}.>`, env, + limits, }); this.init(); return new Proxy(this, { diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 9288e2cfa0..35ae8b8fbd 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -100,6 +100,7 @@ export class OpenFiles { // When a client has a file open, they should periodically // touch it to indicate that it is open. // updates timestamp and ensures open=true. + // id = compute_server_id. touch = async (obj0: { path: string; id?: number }) => { // just read and write it back, which updates the timestamp // no encode/decode needed. diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 1f50ca611f..31eb6335b0 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -77,7 +77,8 @@ interface FilteredStreamLimitOptions { max_msgs: number; // Maximum age of any message in the stream matching the filter, // expressed in nanoseconds. 0 for unlimited. - // Use 'import {nanos} from "@cocalc/nats/util"' then "nanos(milliseconds)" to give input in ms. + // Use 'import {nanos} from "@cocalc/nats/util"' then "nanos(milliseconds)" + // to give input in milliseconds. max_age: Nanos; // How big the Stream may be, when the combined stream size matching the filter // exceeds this old messages are removed. -1 for unlimited. @@ -231,7 +232,7 @@ export class Stream extends EventEmitter { data.length > this.limits.max_msg_size ) { throw Error( - `message size exceeds max_msg_size=${this.limits.max_msg_size} bytes`, + `message size (=${data.length}) exceeds max_msg_size=${this.limits.max_msg_size} bytes`, ); } this.enforceLimits(); diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index fa010a7f68..bcf0a05747 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -109,4 +109,3 @@ export function nanos(millis: number): Nanos { export function millis(ns: Nanos): number { return Math.floor(ns / 1000000); } - From 0991640f54d3c91d396b50450de2549a6deae897 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 14:43:34 +0000 Subject: [PATCH 154/281] nats dkv: implement limits; also handling of rejected changes due to size --- src/packages/nats/sync/dkv.ts | 38 ++++++++++++++++++++++++++- src/packages/nats/sync/general-dkv.ts | 24 ++++++++++++----- src/packages/nats/sync/general-kv.ts | 29 +++++++++++++++++--- 3 files changed, 79 insertions(+), 12 deletions(-) diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index bbfd0a21f9..de796be160 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -28,7 +28,14 @@ export class DKV extends EventEmitter { private prefix: string; private sha1; - constructor({ name, account_id, project_id, merge, env }: DKVOptions) { + constructor({ + name, + account_id, + project_id, + merge, + env, + limits, + }: DKVOptions) { super(); if (env == null) { throw Error("env must not be null"); @@ -43,6 +50,7 @@ export class DKV extends EventEmitter { filter: `${this.prefix}.>`, env, merge, + limits, }); this.init(); return new Proxy(this, { @@ -72,6 +80,19 @@ export class DKV extends EventEmitter { if (this.generalDKV == null) { throw Error("closed"); } + this.generalDKV.on("change", (_, value, prev) => { + if (value != null) { + this.emit("change", value.key, value.value); + } else if (prev != null) { + // value is null so it's a delete + this.emit("change", prev.key); + } + }); + this.generalDKV.on("reject", (_, value) => { + if (value != null) { + this.emit("reject", value.key, value.value); + } + }); await this.generalDKV.init(); }); @@ -115,6 +136,21 @@ export class DKV extends EventEmitter { } this.generalDKV.set(`${this.prefix}.${this.sha1(key)}`, { key, value }); }; + + hasUnsavedChanges = () => { + if (this.generalDKV == null) { + return false; + } + return this.generalDKV.hasUnsavedChanges(); + }; + + unsavedChanges = () => { + const generalDKV = this.generalDKV; + if (generalDKV == null) { + return []; + } + return generalDKV.unsavedChanges().map((key) => generalDKV.get(key)?.key); + }; } const cache: { [key: string]: DKV } = {}; diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index d368013775..33cee106e9 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -71,7 +71,7 @@ TODO: */ import { EventEmitter } from "events"; -import { GeneralKV } from "./general-kv"; +import { GeneralKV, type KVLimits } from "./general-kv"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { type NatsEnv } from "@cocalc/nats/types"; import { isEqual } from "lodash"; @@ -98,6 +98,7 @@ export class GeneralDKV extends EventEmitter { filter, merge, options, + limits, }: { name: string; env: NatsEnv; @@ -112,10 +113,12 @@ export class GeneralDKV extends EventEmitter { // NOTE: any key name that you *set or delete* must match one of these filter: string | string[]; options?; + limits?: KVLimits; }) { super(); this.merge = merge; - this.kv = new GeneralKV({ name, env, filter, options }); + //this.limits = limits; + this.kv = new GeneralKV({ name, env, filter, options, limits }); } init = reuseInFlight(async () => { @@ -144,13 +147,13 @@ export class GeneralDKV extends EventEmitter { private handleRemoteChange = (key, remote, ancestor) => { const local = this.local[key]; + let value: any = remote; if (local !== undefined) { if (isEqual(local, remote)) { // we have a local change, but it's the same change as remote, so just // forget about our local change. delete this.local[key]; } else { - let value; try { value = this.merge?.({ key, local, remote, ancestor }); } catch { @@ -167,7 +170,7 @@ export class GeneralDKV extends EventEmitter { } } } - this.emit("change", key); + this.emit("change", key, value, ancestor); }; get = (key?) => { @@ -239,8 +242,7 @@ export class GeneralDKV extends EventEmitter { await this.attemptToSave(); //console.log("successfully saved"); } catch { - //(err) { - // console.log("problem saving", err); + // console.log("temporary issue saving") } if (this.hasUnsavedChanges()) { d = Math.min(10000, d * 1.3) + Math.random() * 100; @@ -266,7 +268,15 @@ export class GeneralDKV extends EventEmitter { } } } - await this.kv.set(obj); + try { + await this.kv.set(obj); + } catch (err) { + if (err.code == "REJECT" && err.key) { + this.emit("reject", err.key, this.local[err.key]); + delete this.local[err.key]; + } + throw err; + } for (const key in obj) { if (obj[key] === this.local[key] && !this.changed.has(key)) { delete this.local[key]; diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index 29e7e147c2..6c469d3250 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -106,6 +106,11 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { map as awaitMap } from "awaiting"; import { throttle } from "lodash"; +class RejectError extends Error { + code: string; + key: string; +} + const MAX_PARALLEL = 50; // Note that the limit options are named in exactly the same was as for streams, @@ -398,13 +403,29 @@ export class GeneralKV extends EventEmitter { this.limits.max_msg_size > -1 && val.length > this.limits.max_msg_size ) { - throw Error( + // we reject due to our own size reasons + const err = new RejectError( `message key:value size (=${val.length}) exceeds max_msg_size=${this.limits.max_msg_size} bytes`, ); + err.code = "REJECT"; + err.key = key; + throw err; + } + try { + await this.kv.put(key, val, { + previousSeq: revision, + }); + } catch (err) { + if (err.code == "MAX_PAYLOAD_EXCEEDED") { + // nats rejects due to payload size + const err2 = new RejectError(`${err}`); + err2.code = "REJECT"; + err2.key = key; + throw err2; + } else { + throw err; + } } - await this.kv.put(key, val, { - previousSeq: revision, - }); }; // ensure any limits are satisfied, always by deleting old keys From fa07410246c72b77f40d61b1d6ee5d08987493ce Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 15:55:32 +0000 Subject: [PATCH 155/281] nats dstream: add reject event support --- src/packages/nats/sync/dkv.ts | 10 +++--- src/packages/nats/sync/dstream.ts | 18 +++++++--- src/packages/nats/sync/general-dkv.ts | 48 ++++++++++++++++----------- src/packages/nats/sync/general-kv.ts | 2 +- src/packages/nats/sync/stream.ts | 28 ++++++++++++++-- 5 files changed, 74 insertions(+), 32 deletions(-) diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index de796be160..d8180570e8 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -80,17 +80,17 @@ export class DKV extends EventEmitter { if (this.generalDKV == null) { throw Error("closed"); } - this.generalDKV.on("change", (_, value, prev) => { + this.generalDKV.on("change", ({ value, prev }) => { if (value != null) { - this.emit("change", value.key, value.value); + this.emit("change", { key: value.key, value: value.value }); } else if (prev != null) { // value is null so it's a delete - this.emit("change", prev.key); + this.emit("change", { key: prev.key }); } }); - this.generalDKV.on("reject", (_, value) => { + this.generalDKV.on("reject", ({ value }) => { if (value != null) { - this.emit("reject", value.key, value.value); + this.emit("reject", { key: value.key, value: value.value }); } }); await this.generalDKV.init(); diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index cd1049600b..e04140a194 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -156,11 +156,21 @@ export class DStream extends EventEmitter { throw Error("closed"); } const { mesg, subject } = this.local[id]; - // @ts-ignore - await this.stream.publish(mesg, subject, { msgID: id }); - delete this.local[id]; + try { + // @ts-ignore + await this.stream.publish(mesg, subject, { msgID: id }); + delete this.local[id]; + } catch (err) { + if (err.code == "REJECT") { + delete this.local[id]; + this.emit("reject", err.mesg, err.subject); + } else { + throw err; + } + } }; - // NOTE: ES6 spec guarantees "String keys are returned in the order in which they were added to the object." + // NOTE: ES6 spec guarantees "String keys are returned in the order + // in which they were added to the object." await awaitMap(Object.keys(this.local), MAX_PARALLEL, f); }); diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index 33cee106e9..90f0f0de88 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -29,13 +29,13 @@ is a DKV, you can also access the underlying KV via "store.kv". - call "store.unsavedChanges()" to see the unsaved keys. -- The 3-way merge function takes as input {local,remote,ancestor,key}, where +- The 3-way merge function takes as input {local,remote,prev,key}, where - key = the key where there's a conflict - local = your version of the value - remote = the remote value, which conflicts in that isEqual(local,remote) is false. - - ancestor = a known common ancestor of local and remote. + - prev = a known common prev of local and remote. - (any of local, remote or ancestor can be undefined, e.g., no previous value or a key was deleted) + (any of local, remote or prev can be undefined, e.g., no previous value or a key was deleted) You can do anything synchronously you want to resolve such conflicts, i.e., there are no axioms that have to be satisifed. If the 3-way merge function throws an exception (or is @@ -76,12 +76,14 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { type NatsEnv } from "@cocalc/nats/types"; import { isEqual } from "lodash"; import { delay } from "awaiting"; +import { map as awaitMap } from "awaiting"; const TOMBSTONE = Symbol("tombstone"); +const MAX_PARALLEL = 50; export type MergeFunction = (opts: { key: string; - ancestor: any; + prev: any; local: any; remote: any; }) => any; @@ -105,7 +107,7 @@ export class GeneralDKV extends EventEmitter { // 3-way merge conflict resolution merge?: (opts: { key: string; - ancestor?: any; + prev?: any; local?: any; remote?: any; }) => any; @@ -145,7 +147,7 @@ export class GeneralDKV extends EventEmitter { delete this.merge; }; - private handleRemoteChange = (key, remote, ancestor) => { + private handleRemoteChange = ({ key, remote, prev }) => { const local = this.local[key]; let value: any = remote; if (local !== undefined) { @@ -155,7 +157,7 @@ export class GeneralDKV extends EventEmitter { delete this.local[key]; } else { try { - value = this.merge?.({ key, local, remote, ancestor }); + value = this.merge?.({ key, local, remote, prev }); } catch { // user provided a merge function that throws an exception. We select local, since // it is the newest, i.e., "last write wins" @@ -170,7 +172,7 @@ export class GeneralDKV extends EventEmitter { } } } - this.emit("change", key, value, ancestor); + this.emit("change", { key, value, prev }); }; get = (key?) => { @@ -268,19 +270,25 @@ export class GeneralDKV extends EventEmitter { } } } - try { - await this.kv.set(obj); - } catch (err) { - if (err.code == "REJECT" && err.key) { - this.emit("reject", err.key, this.local[err.key]); - delete this.local[err.key]; + const f = async (key) => { + if (this.kv == null) { + // closed + return; } - throw err; - } - for (const key in obj) { - if (obj[key] === this.local[key] && !this.changed.has(key)) { - delete this.local[key]; + try { + await this.kv.set(key, obj[key]); + if (obj[key] === this.local[key] && !this.changed.has(key)) { + // successfully saved this + delete this.local[key]; + } + } catch (err) { + if (err.code == "REJECT" && err.key) { + delete this.local[err.key]; // can never save this. + this.emit("reject", { key: err.key, value: this.local[err.key] }); + } + throw err; } - } + }; + await awaitMap(Object.keys(obj), MAX_PARALLEL, f); }); } diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index 6c469d3250..489ef2f3d9 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -234,7 +234,7 @@ export class GeneralKV extends EventEmitter { this.times[key] = sm.time; this.sizes[key] = value.length; } - this.emit("change", key, this.all[key], prev); + this.emit("change", { key, value: this.all[key], prev }); this.enforceLimits(); } }; diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 31eb6335b0..f7d6716bbb 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -50,6 +50,12 @@ import { isNumericString } from "@cocalc/util/misc"; import { map as awaitMap } from "awaiting"; import { sha1 } from "@cocalc/util/misc"; +class PublishRejectError extends Error { + code: string; + mesg: any; + subject?: string; +} + const MAX_PARALLEL = 50; // confirm that ephemeral consumer still exists every 15 seconds: @@ -231,12 +237,30 @@ export class Stream extends EventEmitter { this.limits.max_msg_size > -1 && data.length > this.limits.max_msg_size ) { - throw Error( + const err = new PublishRejectError( `message size (=${data.length}) exceeds max_msg_size=${this.limits.max_msg_size} bytes`, ); + err.code = "REJECT"; + err.mesg = mesg; + err.subject = subject; + throw err; } this.enforceLimits(); - const resp = await this.js.publish(subject ?? this.subject, data, options); + let resp; + try { + resp = await this.js.publish(subject ?? this.subject, data, options); + } catch (err) { + if (err.code == "MAX_PAYLOAD_EXCEEDED") { + // nats rejects due to payload size + const err2 = new PublishRejectError(`${err}`); + err2.code = "REJECT"; + err2.mesg = mesg; + err2.subject = subject; + throw err2; + } else { + throw err; + } + } this.enforceLimits(); return resp; }; From df984813f069a8ca2d1aea3622dbc009b3c17adb Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 16:21:08 +0000 Subject: [PATCH 156/281] nats: add better time support to kv and stream --- src/packages/nats/sync/dkv.ts | 10 ++++ src/packages/nats/sync/dstream.ts | 9 ++++ src/packages/nats/sync/general-dkv.ts | 7 +++ src/packages/nats/sync/general-kv.ts | 2 +- src/packages/nats/sync/kv.ts | 10 ++++ src/packages/nats/sync/open-files.ts | 61 +++++++++++++++++++++++++ src/packages/nats/sync/stream.ts | 13 +++++- src/packages/project/nats/open-files.ts | 3 +- src/packages/project/nats/sync.ts | 8 ++-- 9 files changed, 115 insertions(+), 8 deletions(-) diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index d8180570e8..c422f8387d 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -113,6 +113,16 @@ export class DKV extends EventEmitter { this.generalDKV.delete(`${this.prefix}.${this.sha1(key)}`); }; + // server assigned time + time = (key?: string) => { + if (this.generalDKV == null) { + throw Error("closed"); + } + return this.generalDKV.time( + key ? `${this.prefix}.${this.sha1(key)}` : undefined, + ); + }; + get = (key?) => { if (this.generalDKV == null) { throw Error("closed"); diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index e04140a194..ccfceb54a4 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -30,6 +30,7 @@ import { delay } from "awaiting"; import { map as awaitMap } from "awaiting"; import { isNumericString } from "@cocalc/util/misc"; import { sha1 } from "@cocalc/util/misc"; +import { millis } from "@cocalc/nats/util"; const MAX_PARALLEL = 50; @@ -101,6 +102,14 @@ export class DStream extends EventEmitter { return this.raw[n]?.seq; }; + time = (n) => { + const r = this.raw[n]; + if (r == null) { + return; + } + return new Date(millis(r?.info.timestampNanos)); + }; + get length() { return this.messages.length + Object.keys(this.local).length; } diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index 90f0f0de88..0e8976af64 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -196,6 +196,13 @@ export class GeneralDKV extends EventEmitter { return x; }; + time = (key?: string) => { + if (this.kv == null) { + throw Error("closed"); + } + return this.kv.time(key); + }; + private assertValidKey = (key) => { if (this.kv == null) { throw Error("closed"); diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index 489ef2f3d9..d5327c7a13 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -260,7 +260,7 @@ export class GeneralKV extends EventEmitter { } }; - time = (key?) => { + time = (key?: string) => { if (key == null) { return this.times; } else { diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index 22ede8d5fe..042955e7fe 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -101,6 +101,16 @@ export class KV extends EventEmitter { await this.generalKV.clear(); }; + // server assigned time + time = (key?: string) => { + if (this.generalKV == null) { + throw Error("closed"); + } + return this.generalKV.time( + key ? `${this.prefix}.${this.sha1(key)}` : undefined, + ); + }; + get = (key?) => { if (this.generalKV == null) { throw Error("closed"); diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 35ae8b8fbd..00da284873 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -232,3 +232,64 @@ export class OpenFiles { }; }; } + +import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; +export class OpenFiles2 { + private project_id: string; + private env: NatsEnv; + private dkv?: DKV; + + constructor({ env, project_id }: { env: NatsEnv; project_id: string }) { + this.env; + this.project_id = project_id; + } + + init = async () => { + this.dkv = await dkv({ + name: "open-files", + project_id: this.project_id, + env: this.env, + }); + }; + + close = () => { + if (this.dkv == null) { + return; + } + this.dkv.close(); + delete this.dkv; + }; + + // When a client has a file open, they should periodically + // touch it to indicate that it is open. + // updates timestamp and ensures open=true. + // do we need compute server? +// touch = async ({ path }: { path: string }) => { +// const { dkv } = this; +// if (dkv == null) { +// throw Error("closed"); +// } +// cur = dkv.get(path); +// const newValue = { ...cur, path }; + +// // just read and write it back, which updates the timestamp +// // no encode/decode needed. +// const obj = { ...validObject(obj0), open: true }; +// const key = this.getKey(obj); +// const kv = await this.getKv(); +// const mesg = await kv.get(key); +// if (mesg == null || mesg.sm.data.length == 0) { +// // no current entry -- create new +// await this.set(obj); +// } else { +// const cur = this.decode(mesg, true); +// const newValue = { ...cur, ...obj }; +// if (!isEqual(cur, newValue)) { +// await this.set(newValue); +// } else { +// // update existing by just rewriting it back; this updates timestamp too +// await kv.put(key, mesg.sm.data); +// } +// } +// }; +} diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index f7d6716bbb..9298348a14 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -43,7 +43,7 @@ import { type NatsEnv } from "@cocalc/nats/types"; import { jetstreamManager, jetstream } from "@nats-io/jetstream"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { jsName, streamSubject } from "@cocalc/nats/names"; -import { nanos, type Nanos } from "@cocalc/nats/util"; +import { nanos, type Nanos, millis } from "@cocalc/nats/util"; import { delay } from "awaiting"; import { throttle } from "lodash"; import { isNumericString } from "@cocalc/util/misc"; @@ -215,11 +215,20 @@ export class Stream extends EventEmitter { } }; - // get sequence number of n-th message in stream + // get server assigned global sequence number of n-th message in stream seq = (n) => { return this.raw[n]?.seq; }; + // get server assigned time of n-th message in stream + time = (n): Date | undefined => { + const r = this.raw[n]; + if (r == null) { + return; + } + return new Date(millis(r?.info.timestampNanos)); + }; + get length() { return this.messages.length; } diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 234ef1ceba..9ee04363d0 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -8,9 +8,10 @@ DEVELOPMENT: await cc.client.nats_client.projectApi({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}).system.terminate({service:'open-files'}) -Set env variables as in a project, then: +Set env variables as in a project (see api/index.ts ), then: > require("@cocalc/project/nats/open-files").init() + */ import { OpenFiles, Entry } from "@cocalc/nats/sync/open-files"; diff --git a/src/packages/project/nats/sync.ts b/src/packages/project/nats/sync.ts index 044b10e2a0..c06675edbb 100644 --- a/src/packages/project/nats/sync.ts +++ b/src/packages/project/nats/sync.ts @@ -10,18 +10,18 @@ import { project_id } from "@cocalc/project/data"; export type { Stream, DStream, KV, DKV }; -export async function stream(opts) { +export async function stream(opts): Promise { return await createStream({ project_id, env: await getEnv(), ...opts }); } -export async function dstream(opts) { +export async function dstream(opts): Promise { return await createDstream({ project_id, env: await getEnv(), ...opts }); } -export async function kv(opts) { +export async function kv(opts): Promise { return await createKV({ project_id, env: await getEnv(), ...opts }); } -export async function dkv(opts) { +export async function dkv(opts): Promise { return await createDKV({ project_id, env: await getEnv(), ...opts }); } From 0a7eb655ca7a872dfb893ca59352553ecb18f594 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 17:20:27 +0000 Subject: [PATCH 157/281] nats: rewrite open files tracker to use new dkv --- src/packages/frontend/client/client.ts | 4 +- src/packages/frontend/nats/client.ts | 7 +- src/packages/nats/sync/dkv.ts | 12 +- src/packages/nats/sync/general-dkv.ts | 2 +- src/packages/nats/sync/open-files.ts | 325 +++++------------- src/packages/nats/sync/pubsub.ts | 3 +- src/packages/nats/sync/synctable-kv-atomic.ts | 3 +- src/packages/nats/types.ts | 3 + src/packages/project/nats/open-files.ts | 24 +- 9 files changed, 123 insertions(+), 260 deletions(-) diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 49eb0b66a6..49deb4f411 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -348,14 +348,14 @@ class Client extends EventEmitter implements WebappClient { touchOpenFile = async ({ project_id, path, - id, + // id, }: { project_id: string; path: string; id?: number; }) => { const x = await this.nats_client.openFiles(project_id); - await x.touch({ path, id }); + await x.touch({ path }); }; } diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 51f0753921..18534deb08 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -15,7 +15,7 @@ import { type ProjectApi, initProjectApi } from "@cocalc/nats/project-api"; import { type BrowserApi, initBrowserApi } from "@cocalc/nats/browser-api"; import { getPrimusConnection } from "@cocalc/nats/primus"; import { isValidUUID } from "@cocalc/util/misc"; -import { OpenFiles } from "@cocalc/nats/sync/open-files"; +import { createOpenFiles, OpenFiles } from "@cocalc/nats/sync/open-files"; import { PubSub } from "@cocalc/nats/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; import { kv, type KVOptions } from "@cocalc/nats/sync/kv"; @@ -333,10 +333,13 @@ export class NatsClient { openFiles = reuseInFlight(async (project_id: string) => { if (this.openFilesCache[project_id] == null) { - this.openFilesCache[project_id] = new OpenFiles({ + this.openFilesCache[project_id] = await createOpenFiles({ project_id, env: await this.getEnv(), }); + this.openFilesCache[project_id].on("closed", () => { + delete this.openFilesCache[project_id]; + }); } return this.openFilesCache[project_id]!; }); diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index c422f8387d..938d2a02f1 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -118,9 +118,19 @@ export class DKV extends EventEmitter { if (this.generalDKV == null) { throw Error("closed"); } - return this.generalDKV.time( + const times = this.generalDKV.time( key ? `${this.prefix}.${this.sha1(key)}` : undefined, ); + if (key != null || times == null) { + return times; + } + const obj = this.generalDKV.get(); + const x: any = {}; + for (const k in obj) { + const { key } = obj[k]; + x[key] = times[k]; + } + return x; }; get = (key?) => { diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index 0e8976af64..443cc3ef2d 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -147,7 +147,7 @@ export class GeneralDKV extends EventEmitter { delete this.merge; }; - private handleRemoteChange = ({ key, remote, prev }) => { + private handleRemoteChange = ({ key, value: remote, prev }) => { const local = this.local[key]; let value: any = remote; if (local !== undefined) { diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 00da284873..beed0cac0c 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -3,44 +3,37 @@ NATS Kv associated to a project to keep track of open files. DEVELOPMENT: -Change to packages/project, since packages/nats doesn't have a way to connect: - -~/cocalc/src/packages/project$ node -> z = new (require('@cocalc/nats/sync/open-files').OpenFiles)({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf', env:await require('@cocalc/backend/nats').getEnv()}) -> await z.touch({path:'a.txt',id:1}) -> await z.get() -{ - 'a.txt': { path: 'a.txt', open: true, id: 1, time: 2025-01-31T14:16:48.314Z } -} -> await z.touch({path:'foo/b.md',id:0}) -undefined -> await z.get() +Change to packages/backend, since packages/nats doesn't have a way to connect: + +~/cocalc/src/packages/backend node +> z = await require('@cocalc/nats/sync/open-files').createOpenFiles({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf', env:await require('@cocalc/backend/nats').getEnv()}) +> z.touch({path:'a.txt'}) +> z.get({path:'a.txt'}) +{ open: true, count: 1 } +> z.touch({path:'a.txt'}) +> z.get({path:'a.txt'}) +{ open: true, count: 2 } +> z.time({path:'a.txt'}) +2025-02-09T16:36:58.510Z +> z.touch({path:'foo/b.md',id:0}) +> z.get() { - 'a.txt': { path: 'a.txt', interest: 1738298844728, open: true, id: 1, time: 2025-01-31T14:16:48.314Z }, - 'foo/b.md': { path: 'foo/b.md', interest: 1738298896539, open: true, id: 0, time:... } + 'a.txt': { open: true, count: 3 }, + 'foo/b.md': { open: true, count: 1 } } -> await z.get({path:'foo/b.dm'}) -null -> await z.get({path:'foo/b.md'}) -{ path: 'foo/b.md', open: true, id: 0 } - -> for await (const x of await z.watch()) { console.log(x)} - */ -import { type NatsEnv } from "@cocalc/nats/types"; -import { getKv } from "./synctable-kv"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { sha1 } from "@cocalc/util/misc"; -import { isEqual } from "lodash"; +import { type NatsEnv, type State } from "@cocalc/nats/types"; +import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; +import { nanos } from "@cocalc/nats/util"; +import { EventEmitter } from "events"; -const PREFIX = `open-files`; +// 1 week +const MAX_AGE_MS = 1000 * 60 * 60 * 168; export interface Entry { // path to file relative to HOME path: string; - // compute server id or 0/not defined for home base - id?: number; // if true, then file should be opened, managed, and watched // by home base or compute server open?: boolean; @@ -49,247 +42,99 @@ export interface Entry { // https://github.com/nats-io/nats-server/discussions/3095 // It gets updated even if you set an object to itself (making no change). time?: Date; + // + count?: number; } -const FIELDS = ["path", "id", "open"]; - -function validObject(obj: Entry) { - const obj2: any = {}; - for (const field of FIELDS) { - const val = obj[field]; - if (val != null) { - if (field == "path") { - obj2[field] = typeof val == "string" ? val : `${val}`; - } else if (field == "id") { - obj2[field] = typeof val == "number" ? val : parseInt(`${val}`); - } else if (field == "open") { - obj2[field] = typeof val == "boolean" ? val : !!val; - } - } - } - if (!obj2["path"]) { - throw Error("path must be specified"); - } - return obj2; -} - -export class OpenFiles { - private kv?; - private nc; - private jc; - private sha1; - private project_id: string; - public state: "ready" | "closed" = "ready"; - private watches: any[] = []; - - constructor({ env, project_id }: { env: NatsEnv; project_id: string }) { - this.sha1 = env.sha1 ?? sha1; - this.nc = env.nc; - this.jc = env.jc; - this.project_id = project_id; - } - - close = () => { - this.state = "closed"; - for (const w of this.watches) { - w.stop(); - } - this.watches = []; - }; - - // When a client has a file open, they should periodically - // touch it to indicate that it is open. - // updates timestamp and ensures open=true. - // id = compute_server_id. - touch = async (obj0: { path: string; id?: number }) => { - // just read and write it back, which updates the timestamp - // no encode/decode needed. - const obj = { ...validObject(obj0), open: true }; - const key = this.getKey(obj); - const kv = await this.getKv(); - const mesg = await kv.get(key); - if (mesg == null || mesg.sm.data.length == 0) { - // no current entry -- create new - await this.set(obj); - } else { - const cur = this.decode(mesg, true); - const newValue = { ...cur, ...obj }; - if (!isEqual(cur, newValue)) { - await this.set(newValue); - } else { - // update existing by just rewriting it back; this updates timestamp too - await kv.put(key, mesg.sm.data); - } - } - }; - - closeFile = async ({ path }: { path: string }) => { - const kv = await this.getKv(); - const key = this.getKey({ path }); - const mesg = await kv.get(key); - if (mesg.sm.data.length == 0) { - // nothing to do - return; - } - const cur = this.decode(mesg, true); - const value = this.jc.encode({ ...cur, open: false }); - await kv.put(key, value); - }; - - get = async (obj?: Entry) => { - const kv = await this.getKv(); - if (obj == null) { - // everything - const keys = await kv.keys(`${PREFIX}.>`); - const all: { [path: string]: Entry } = {}; - for await (const key of keys) { - const obj = this.decode(await kv.get(key)); - all[obj.path] = obj; - } - return all; - } - return this.decode(await kv.get(this.getKey(validObject(obj)))); - }; - - // watch for changes - async *watch() { - const kv = await this.getKv(); - const w = await kv.watch({ - key: `${PREFIX}.>`, - // we assume that we ONLY delete old items which are not relevant - ignoreDeletes: true, - }); - this.watches.push(w); - for await (const mesg of w) { - // no need to check for 'mesg.value.length' due to ignoreDeletes above. - yield this.decode(mesg); - if (this.state == "closed") { - return; - } - } - } - - // delete entries that haven't been touched in ageMs milliseconds. - // default=a month - // returns number of deleted objects. - expire = async (ageMs: number = 1000 * 60 * 60 * 730): Promise => { - let n = 0; - const cutoff = new Date(Date.now() - ageMs); - const kv = await this.getKv(); - const keys = await kv.keys(`${PREFIX}.>`); - for await (const key of keys) { - const mesg = await kv.get(key); - if (mesg.sm.time <= cutoff) { - await kv.delete(key); - n += 1; - } - } - return n; - }; - - // dangerous - e.g., our watcher assumes no deletes. Instead, you should - // close files, not delete. - delete = async (obj0) => { - const obj = validObject(obj0); - const kv = await this.getKv(); - await kv.delete(this.getKey(obj)); - }; - - has = async ({ path }): Promise => { - const kv = await this.getKv(); - const key = this.getKey({ path }); - const mesg = await kv.get(key); - return mesg.sm.data.length > 0; - }; - - private getKv = reuseInFlight(async () => { - if (this.kv == null) { - this.kv = await getKv({ - nc: this.nc, - project_id: this.project_id, - }); - } - return this.kv!; - }); - - private getKey = ({ path }: Entry): string => { - return `${PREFIX}.${this.sha1(path)}`; - }; - - // atomic set - NOT a merge set. - private set = async (obj0: Entry) => { - let obj = validObject(obj0); - const key = this.getKey(obj); - const value = this.jc.encode(obj); - const kv = await this.getKv(); - await kv.put(key, value); - }; - - private decode = (mesg, noDate = false): Entry => { - return { - ...this.jc.decode(mesg.sm.data), - ...(noDate ? undefined : { time: mesg.sm.time }), - }; - }; +export async function createOpenFiles({ env, project_id }) { + const openFiles = new OpenFiles({ env, project_id }); + await openFiles.init(); + return openFiles; } -import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; -export class OpenFiles2 { +export class OpenFiles extends EventEmitter { private project_id: string; private env: NatsEnv; private dkv?: DKV; + public state: "disconnected" | "connected" | "closed" = "disconnected"; constructor({ env, project_id }: { env: NatsEnv; project_id: string }) { - this.env; + super(); + this.env = env; this.project_id = project_id; } + private setState = (state: State) => { + this.state = state; + this.emit(state); + }; + init = async () => { - this.dkv = await dkv({ + const d = await dkv({ name: "open-files", project_id: this.project_id, env: this.env, + limits: { + max_age: nanos(MAX_AGE_MS), + }, }); + this.dkv = d; + d.on("change", ({ key: path, value }) => { + const time = d.time(path); + const { open } = value ?? {}; + this.emit("change", { path, open, time } as Entry); + }); + this.setState("connected"); }; close = () => { if (this.dkv == null) { return; } + this.setState("closed"); + this.removeAllListeners(); this.dkv.close(); delete this.dkv; + // @ts-ignore + delete this.env; + // @ts-ignore + delete this.project_id; + }; + + private getDkv = () => { + const { dkv } = this; + if (dkv == null) { + throw Error("closed"); + } + return dkv; }; // When a client has a file open, they should periodically // touch it to indicate that it is open. // updates timestamp and ensures open=true. // do we need compute server? -// touch = async ({ path }: { path: string }) => { -// const { dkv } = this; -// if (dkv == null) { -// throw Error("closed"); -// } -// cur = dkv.get(path); -// const newValue = { ...cur, path }; + touch = ({ path }: { path: string }) => { + const dkv = this.getDkv(); + // n = sequence number to make sure a write happens, which updates + // server assigned timestamp. + const count = dkv.get(path)?.count ?? 0; + dkv.set(path, { open: true, count: count + 1 }); + }; + + closeFile = ({ path }: { path: string }) => { + const dkv = this.getDkv(); + dkv.set(path, { ...dkv.get(path), open: false }); + }; + + get = (obj?: { path: string }) => { + return this.getDkv().get(obj?.path); + }; + + delete = ({ path }: { path: string }) => { + this.getDkv().delete(path); + }; -// // just read and write it back, which updates the timestamp -// // no encode/decode needed. -// const obj = { ...validObject(obj0), open: true }; -// const key = this.getKey(obj); -// const kv = await this.getKv(); -// const mesg = await kv.get(key); -// if (mesg == null || mesg.sm.data.length == 0) { -// // no current entry -- create new -// await this.set(obj); -// } else { -// const cur = this.decode(mesg, true); -// const newValue = { ...cur, ...obj }; -// if (!isEqual(cur, newValue)) { -// await this.set(newValue); -// } else { -// // update existing by just rewriting it back; this updates timestamp too -// await kv.put(key, mesg.sm.data); -// } -// } -// }; + time = (obj?: { path: string }) => { + return this.getDkv().time(obj?.path); + }; } diff --git a/src/packages/nats/sync/pubsub.ts b/src/packages/nats/sync/pubsub.ts index d839f3e39f..60983103e1 100644 --- a/src/packages/nats/sync/pubsub.ts +++ b/src/packages/nats/sync/pubsub.ts @@ -3,9 +3,8 @@ Use NATS simple pub/sub to share state for something *ephemeral* in a project. */ import { projectSubject } from "@cocalc/nats/names"; -import { type NatsEnv } from "@cocalc/nats/types"; +import { type NatsEnv, State } from "@cocalc/nats/types"; import { EventEmitter } from "events"; -import { State } from "./synctable-kv-atomic"; export class PubSub extends EventEmitter { private subject: string; diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index c08119e6fa..d98191330d 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -26,11 +26,10 @@ in the store. import { keys } from "lodash"; import { client_db } from "@cocalc/util/db-schema/client-db"; import { getKv, toKey, natsKeyPrefix } from "./synctable-kv"; -import { type NatsEnv } from "@cocalc/nats/types"; +import { type NatsEnv, State } from "@cocalc/nats/types"; import { sha1 } from "@cocalc/util/misc"; import { EventEmitter } from "events"; import { getAllFromKv } from "@cocalc/nats/util"; -export type State = "disconnected" | "connected" | "closed"; export class SyncTableKVAtomic extends EventEmitter { private kv?; diff --git a/src/packages/nats/types.ts b/src/packages/nats/types.ts index 4bafa1cb2a..b981f18437 100644 --- a/src/packages/nats/types.ts +++ b/src/packages/nats/types.ts @@ -4,3 +4,6 @@ export interface NatsEnv { // compute sha1 hash efficiently (set differently on backend) sha1?: (string) => string; } + +export type State = "disconnected" | "connected" | "closed"; + diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 9ee04363d0..3da4b862a2 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -14,7 +14,11 @@ Set env variables as in a project (see api/index.ts ), then: */ -import { OpenFiles, Entry } from "@cocalc/nats/sync/open-files"; +import { + createOpenFiles, + OpenFiles, + Entry, +} from "@cocalc/nats/sync/open-files"; import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; import { compute_server_id, project_id } from "@cocalc/project/data"; import { getEnv } from "./env"; @@ -33,22 +37,22 @@ let openFiles: OpenFiles | null = null; export async function init() { logger.debug("init"); - openFiles = new OpenFiles({ + openFiles = await createOpenFiles({ project_id, env: await getEnv(), }); runLoop(); } -async function runLoop() { +function runLoop() { logger.debug("starting run loop"); if (openFiles != null) { const entries: { [path: string]: Entry } = {}; closeIgnoredFiles(entries, openFiles); - for await (const entry of await openFiles.watch()) { + openFiles.on("change", (entry) => { entries[entry.path] = entry; - await handleEntry(entry); - } + handleChange(entry); + }); } logger.debug("exiting open files run loop"); } @@ -69,9 +73,10 @@ function getCutoff() { return new Date(Date.now() - 2.5 * NATS_OPEN_FILE_TOUCH_INTERVAL); } -async function handleEntry({ path, id = 0, open, time }: Entry) { +async function handleChange({ path, open, time }: Entry) { const syncDoc = openSyncDocs[path]; const isOpenHere = syncDoc != null; + const id = 0; // todo if (id != compute_server_id) { if (isOpenHere) { // close it here @@ -105,9 +110,9 @@ function supportAutoclose(path: string) { } async function closeIgnoredFiles(entries, openFiles) { - while (openFiles.state == "ready") { + while (openFiles.state == "connected") { await delay(NATS_OPEN_FILE_TOUCH_INTERVAL); - if (openFiles.state != "ready") { + if (openFiles.state != "connected") { return; } logger.debug("closeIgnoredFiles: checking..."); @@ -199,7 +204,6 @@ async function getTypeAndOpts( return s != null; }); } - console.log("s = ", s); const opts: any = {}; let type: string = ""; From 9cd3c073fce6dce3efe68ca487d6fbd635fdded5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 17:43:28 +0000 Subject: [PATCH 158/281] nats: do not throttle dkv and dstream unless there are errors (e.g., disconnected) - motivation: terminals use dstream, and they feel horrible with throttling --- src/packages/nats/sync/dstream.ts | 7 +++---- src/packages/nats/sync/general-dkv.ts | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index ccfceb54a4..3637860cae 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -147,13 +147,12 @@ export class DStream extends EventEmitter { await this.attemptToSave(); //console.log("successfully saved"); } catch { + d = Math.min(10000, d * 1.3) + Math.random() * 100; + await delay(d); //(err) { // console.log("problem saving", err); } - if (this.hasUnsavedChanges()) { - d = Math.min(10000, d * 1.3) + Math.random() * 100; - await delay(d); - } else { + if (!this.hasUnsavedChanges()) { return; } } diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index 443cc3ef2d..a6fe62a2aa 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -251,12 +251,11 @@ export class GeneralDKV extends EventEmitter { await this.attemptToSave(); //console.log("successfully saved"); } catch { - // console.log("temporary issue saving") - } - if (this.hasUnsavedChanges()) { d = Math.min(10000, d * 1.3) + Math.random() * 100; await delay(d); - } else { + // console.log("temporary issue saving") + } + if (!this.hasUnsavedChanges()) { return; } } From f363cd0b55aa3ea3378698a0c77b03a5dc30f154 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 18:39:00 +0000 Subject: [PATCH 159/281] nats: switch to using new dstream for patches table :-) --- src/packages/nats/sync/dstream.ts | 5 +- src/packages/nats/sync/stream.ts | 4 +- src/packages/nats/sync/synctable-stream.ts | 138 ++++++--------------- 3 files changed, 41 insertions(+), 106 deletions(-) diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 3637860cae..5a4002bb52 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -140,7 +140,7 @@ export class DStream extends EventEmitter { return Object.values(this.local); }; - private save = reuseInFlight(async () => { + save = reuseInFlight(async () => { let d = 100; while (true) { try { @@ -171,7 +171,8 @@ export class DStream extends EventEmitter { } catch (err) { if (err.code == "REJECT") { delete this.local[id]; - this.emit("reject", err.mesg, err.subject); + // err has mesg and subject set. + this.emit("reject", err); } else { throw err; } diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 9298348a14..05c351dc1a 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -63,11 +63,11 @@ const MAX_PARALLEL = 50; // ensures we successfully get properly updated. const CONSUMER_MONITOR_INTERVAL = 15 * 1000; -// Have server keep ephemeral consumers alive for an hour. This +// Have server keep ephemeral consumers alive for 5 minutes. This // means even if we drop from the internet for up to an hour, the server // doesn't forget about our consumer. But even if we are forgotten, // the CONSUMER_MONITOR_INTERVAL ensures the event stream correctly works! -const EPHEMERAL_CONSUMER_THRESH = 60 * 60 * 1000; +const EPHEMERAL_CONSUMER_THRESH = 5 * 60 * 1000; // We re-implement exactly the same stream-wide limits that NATS has, // but instead, these are for the stream **with the given filter**. diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts index 909cc69399..498ceb8d93 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/nats/sync/synctable-stream.ts @@ -7,13 +7,13 @@ This is ONLY for the scope of patches in a single project. It uses a NATS stream to store the elements in a well defined order. */ -import { jetstreamManager, jetstream } from "@nats-io/jetstream"; import jsonStableStringify from "json-stable-stringify"; import { keys } from "lodash"; import { cmp_Date, is_array, isValidUUID, sha1 } from "@cocalc/util/misc"; import { client_db } from "@cocalc/util/db-schema/client-db"; import { EventEmitter } from "events"; import { type NatsEnv } from "@cocalc/nats/types"; +import { dstream, DStream } from "./dstream"; export type State = "disconnected" | "connected" | "closed"; @@ -28,26 +28,20 @@ function toKey(x): string | undefined { } export class SyncTableStream extends EventEmitter { - private nc; - private jc; - private sha1; public readonly table; private primaryKeys: string[]; private project_id?: string; - private account_id?: string; - private streamName: string; - private streamSubject: string; private path: string; - private subject: string; private string_id: string; private data: any = {}; - private consumer?; private state: State = "disconnected"; + private env; + private dstream?: DStream; constructor({ query, env, - account_id, + account_id: _account_id, project_id, }: { query; @@ -56,72 +50,46 @@ export class SyncTableStream extends EventEmitter { project_id?: string; }) { super(); - this.sha1 = env.sha1 ?? sha1; - this.nc = env.nc; - this.jc = env.jc; + this.env = env; const table = keys(query)[0]; this.table = table; if (table != "patches") { throw Error("only the patches table is supported"); } this.project_id = project_id ?? query[table][0].project_id; - this.account_id = account_id ?? query[table][0].account_id; if (!isValidUUID(this.project_id)) { throw Error("query MUST specify a valid project_id"); } - if (this.account_id && !isValidUUID(this.account_id)) { - throw Error("query MUST specify a valid account_id"); - } this.path = query[table][0].path; if (!this.path) { throw Error("path MUST be specified"); } - query[table][0].string_id = this.string_id = this.sha1( + query[table][0].string_id = this.string_id = (env.sha1 ?? sha1)( `${this.project_id}${this.path}`, ); - this.streamName = `project-${this.project_id}-${this.table}`; - this.streamSubject = `project.${this.project_id}.${this.table}.>`; - this.subject = `project.${this.project_id}.${this.table}.${query[table][0].string_id}`; this.primaryKeys = client_db.primary_keys(table); } - private createStream = async () => { - const jsm = await jetstreamManager(this.nc); - try { - await jsm.streams.add({ - name: this.streamName, - subjects: [this.streamSubject], - compression: "s2", - }); - } catch (err) { - console.log("createStream", err); - // probably already exists - await jsm.streams.update(this.streamName, { - subjects: [this.streamSubject], - compression: "s2" as any, - }); - } - }; - - private getConsumer = async () => { - const js = jetstream(this.nc); - const jsm = await jetstreamManager(this.nc); - // making an ephemeral consumer - const { name } = await jsm.consumers.add(this.streamName, { - filter_subject: this.subject, - }); - return await js.consumers.get(this.streamName, name); - }; - init = async () => { - await this.createStream(); - this.consumer = await this.getConsumer(); - await this.readData(); - this.set_state("connected"); - this.listenForUpdates(); + const name = `patches-${this.string_id}`; + this.dstream = await dstream({ + name, + project_id: this.project_id, + env: this.env, + }); + this.dstream.on("change", (mesg) => { + this.handle(mesg, true); + }); + this.dstream.on("reject", (err) => { + console.warn("synctable-stream: REJECTED - ", err); + }); + for (const mesg of this.dstream.get()) { + this.handle(mesg, false); + } + this.setState("connected"); }; - private set_state = (state: State): void => { + private setState = (state: State): void => { this.state = state; this.emit(state); }; @@ -137,31 +105,25 @@ export class SyncTableStream extends EventEmitter { getKey = this.primaryString; - private publish = (mesg) => { - // console.log("publishing ", { subject: this.subject, mesg }); - this.nc.publish(this.subject, this.jc.encode(mesg)); - }; - set = (obj) => { // console.log("set", obj); // delete string_id since it is redundant info const key = this.primaryString(obj); if (this.data[key] != null) { - // no changes to existing keys -- just ignore. - // TODO? - // console.log("set - skip", obj); return; } const { string_id, ...obj2 } = obj; // console.log("set - publish", obj); - this.publish(obj2); + if (this.dstream == null) { + throw Error("closed"); + } + this.dstream.push(obj2); }; - private handle = (mesg, changeEvent: boolean) => { + private handle = (obj, changeEvent: boolean) => { if (this.state == "closed") { return true; } - const obj = this.jc.decode(mesg.data); const key = this.primaryString(obj); this.data[key] = { ...obj, time: new Date(obj.time) }; if (this.data[key].prev != null) { @@ -170,35 +132,6 @@ export class SyncTableStream extends EventEmitter { if (changeEvent) { this.emit("change", [key]); } - return false; - }; - - // load initial data - private readData = async () => { - const consumer = this.consumer!; - const messages = await consumer.fetch({ - max_messages: 100000, - expires: 1000, - }); - for await (const mesg of messages) { - if (this.handle(mesg, false)) { - return; - } - if (mesg.info.pending == 0) { - // no further messages - break; - } - } - }; - - // listen for new data - private listenForUpdates = async () => { - const consumer = this.consumer!; - for await (const mesg of await consumer.consume()) { - if (this.handle(mesg, true)) { - return; - } - } }; get = (obj?) => { @@ -236,20 +169,21 @@ export class SyncTableStream extends EventEmitter { // already closed return; } - this.set_state("closed"); + this.setState("closed"); this.removeAllListeners(); - this.consumer?.delete(); - delete this.consumer; + this.dstream?.close(); + delete this.dstream; }; delete = async (_obj) => { throw Error("delete: not implemented for stream synctable"); }; - // no-op because we always immediately publish changes on set. - save = () => {}; + save = () => { + this.dstream?.save(); + }; + has_uncommitted_changes = () => { - // todo - if disconnected (?) - return false; + return this.dstream?.hasUnsavedChanges(); }; } From c1d2f6c48d8087b0b887041ffe22ab43d8cc5f1e Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 9 Feb 2025 20:10:16 +0000 Subject: [PATCH 160/281] nats: broken (!) work in progress switching database changefeeds to new DKV data structure - vastly simpler code on the backend... but doesn't 100% work yet --- src/packages/database/nats/changefeeds.ts | 131 ++-------------- src/packages/nats/sync/general-dkv.ts | 9 +- src/packages/nats/sync/synctable-kv-atomic.ts | 141 +++++------------- src/packages/nats/util.ts | 12 ++ src/packages/server/nats/auth.ts | 10 +- src/packages/sync/table/changefeed-nats.ts | 14 +- 6 files changed, 76 insertions(+), 241 deletions(-) diff --git a/src/packages/database/nats/changefeeds.ts b/src/packages/database/nats/changefeeds.ts index 102d575961..b0e118a636 100644 --- a/src/packages/database/nats/changefeeds.ts +++ b/src/packages/database/nats/changefeeds.ts @@ -6,6 +6,7 @@ 2. Run this + require("@cocalc/database/nats/changefeeds").init() */ @@ -25,15 +26,11 @@ import jsonStableStringify from "json-stable-stringify"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { uuid } from "@cocalc/util/misc"; import { delay } from "awaiting"; -import { debounce } from "lodash"; import { Svcm, type ServiceMsg } from "@nats-io/services"; import { type QueuedIterator } from "nats"; const logger = getLogger("database:nats:changefeeds"); -const DEBOUNCE_SAVE_TO_JETSTREAM = 100; -const MAX_TIME_SAVE_TO_JETSTREAM = 30000; - const jc = JSONCodec(); let api: QueuedIterator | null = null; @@ -155,112 +152,10 @@ const createChangefeed = reuseInFlight( atomic: true, }); await synctable.init(); - - /* - This code is complicated because it has to be. - - 0. Extra work to avoid ever setting a key in the nats kv if - we don't have to. Nat's doesn't do anything to avoid broadcasting - changes, so this is very valuable. For a supercluster it will - be a critical optimization. - - 1. The initial set could - take a long time and still be happening as we get updates - later. Thus we MUST use a work queue to ensure that every - update happens in the order it was received and also after - the initial state is set. - - 2. We keep a map in memory of the current value of all objects - so that in the case of an *update* we do not have to read - the last received value, which would take extra time and be - particularly hard given the queue issue in 1. - - 3. Saving to NATS could obviously fail intermittenly, e.g., if - NATS is down for some reason or there are network issues. - We retry with exponential backoff several times and - finally give up... TODO: user is not informed about this yet. - - */ - - // initalize map with exactly what is *currently* in the nats kv, - // so we can be sure to never set anything we don't need to set. - const map = await synctable.get(null, { natsKeys: true }); - const queue: { - action: "insert" | "update" | "delete"; - obj: object; - }[] = []; - - const deleteMap = (key, obj) => { - if (map[key] !== undefined) { - delete map[key]; - queue.push({ action: "delete", obj }); - } - }; - - const setMap = (key, obj) => { - const cur = map[key]; - // always merge set - const value = { ...cur, ...obj }; - // json so dates compare as strings. Yes, we could make this faster, but this - // is entirely in the server. - if (JSON.stringify(cur) != JSON.stringify(value)) { - map[key] = value; - queue.push({ action: "update", obj: value }); - } - }; - - const synctableSetRows = async (rows) => { - if (rows.length == 0) { - return; - } - const v = rows.map(synctable.set); - // wait for confirmation that sets are done - let d = 2000; - let t = 0; - while ( - t < MAX_TIME_SAVE_TO_JETSTREAM && - changefeedHashes[changes] != null - ) { - const s = Date.now(); - try { - await Promise.all(v); - return; - } catch (err) { - logger.debug(`failed to save updates to NATS -- ${err}`); - await delay(d); - d = Math.min(10000, d * 1.3); - } - t += Date.now() - s; - } - logger.debug( - "WARNING: couldn't save to NATS after many attempts -- we cancel this whole changefeed", - ); - cancelChangefeed(changes); - }; - - const processQueue = debounce( - reuseInFlight(async () => { - const work = [...queue]; - // clear queue - queue.length = 0; - const rows: any[] = []; - for (const { action, obj } of work) { - if (action == "delete") { - // if we hit a delete, we have to handle everything up - // to this point, then do the delete. - await synctableSetRows(rows); - rows.length = 0; - await synctable.delete(obj); - } else { - rows.push(obj); - } - } - // handle anything left (will be everything if no deletes) - await synctableSetRows(rows); - }), - DEBOUNCE_SAVE_TO_JETSTREAM, - { leading: true, trailing: true }, - ); + // if (global.z == null) { + // global.z = {}; + // } + // global.z[synctable.table] = synctable; const handleFirst = ({ cb, err, rows }) => { if (err || rows == null) { @@ -268,9 +163,8 @@ const createChangefeed = reuseInFlight( return; } for (const obj of rows) { - setMap(synctable.getKey(obj), obj); + synctable.set(obj); } - processQueue(); cb(); }; @@ -282,20 +176,13 @@ const createChangefeed = reuseInFlight( // nothing we can do with this return; } - const key = synctable.getKey(new_val ?? old_val); - if (action == "insert") { - setMap(key, new_val); - } else if (action == "update") { - // update -- since atomic have to get the current value; - // this of course assumes there is one process writing to - // this part of the key value store (the atomic business). - setMap(key, new_val); + if (action == "insert" || action == "update") { + synctable.set(new_val); } else if (action == "delete") { - deleteMap(key, old_val); + synctable.delete(old_val); } else if (action == "close") { cancelChangefeed(changes); } - processQueue(); }; const f = (cb) => { diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index a6fe62a2aa..7cd786f802 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -157,7 +157,14 @@ export class GeneralDKV extends EventEmitter { delete this.local[key]; } else { try { - value = this.merge?.({ key, local, remote, prev }); + value = this.merge?.({ key, local, remote, prev }) ?? local; +// console.log("handle merge conflict", { +// key, +// local, +// remote, +// prev, +// value, +// }); } catch { // user provided a merge function that throws an exception. We select local, since // it is the newest, i.e., "last write wins" diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index d98191330d..f1f4a64b3b 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -1,48 +1,27 @@ /* -TODO: This is a kv store where you atomically do updates. The way this is written, two clents -might make a change to the same object at the same time and one overwrites the other. -However, I just realized with NATS we can easily prevent this!! There is a version -option to update, so using that instead of put make it possible to detect if there's a -potential conflict, then fix and retry!!! -(See packages/nats/sync/kv.ts for how to do this properly!) - - * Updates the existing entry provided that the previous sequence - * for the Kv is at the specified version. This ensures that the - * KV has not been modified prior to the update. - * @param k - * @param data - * @param version - update(k: string, data: Payload, version: number): Promise; - - -The synctable-kv.ts file has another one where each key:value in a single object is its own key:value -in the store. */ import { keys } from "lodash"; import { client_db } from "@cocalc/util/db-schema/client-db"; -import { getKv, toKey, natsKeyPrefix } from "./synctable-kv"; -import { type NatsEnv, State } from "@cocalc/nats/types"; -import { sha1 } from "@cocalc/util/misc"; +import type { NatsEnv, State } from "@cocalc/nats/types"; import { EventEmitter } from "events"; -import { getAllFromKv } from "@cocalc/nats/util"; +import { dkv as createDkv, type DKV } from "./dkv"; +import jsonStableStringify from "json-stable-stringify"; +import { toKey } from "@cocalc/nats/util"; export class SyncTableKVAtomic extends EventEmitter { - private kv?; - private nc; - private jc; - private sha1; - public readonly natsKeyPrefix; public readonly table; + private query; private primaryKeys: string[]; private project_id?: string; private account_id?: string; private state: State = "disconnected"; - private watches: any[] = []; + private dkv?: DKV; + private env; constructor({ query, @@ -56,15 +35,12 @@ export class SyncTableKVAtomic extends EventEmitter { project_id?: string; }) { super(); - this.sha1 = env.sha1 ?? sha1; - this.nc = env.nc; - this.jc = env.jc; - const table = keys(query)[0]; - this.table = table; - this.natsKeyPrefix = natsKeyPrefix({ query, atomic: true }); - this.project_id = project_id ?? query[table][0].project_id; - this.account_id = account_id ?? query[table][0].account_id; - this.primaryKeys = client_db.primary_keys(table); + this.query = query; + this.env = env; + this.table = keys(query)[0]; + this.account_id = account_id ?? query[this.table][0].account_id; + this.project_id = project_id; + this.primaryKeys = client_db.primary_keys(this.table); } private set_state = (state: State): void => { @@ -77,15 +53,19 @@ export class SyncTableKVAtomic extends EventEmitter { }; init = async () => { - this.kv = await getKv({ - nc: this.nc, - project_id: this.project_id, + this.dkv = await createDkv({ + name: jsonStableStringify(this.query), account_id: this.account_id, + project_id: this.project_id, + env: this.env, + }); + this.dkv.on("change", (x) => { + this.emit("change", x); }); this.set_state("connected"); }; - private primaryString = (obj): string => { + getKey = (obj): string => { if (this.primaryKeys.length === 1) { return toKey(obj[this.primaryKeys[0]] ?? "")!; } else { @@ -94,80 +74,31 @@ export class SyncTableKVAtomic extends EventEmitter { } }; - private natObjectKey = (obj): string => { - if (obj == null) { - throw Error("obj must be an object (not null)"); - } - return this.sha1(this.primaryString(obj)); - }; - - getKey = (obj): string => { - return `${this.natsKeyPrefix}.${this.natObjectKey(obj)}`; + set = (obj) => { + if (this.dkv == null) throw Error("closed"); + this.dkv.set(this.getKey(obj), obj); }; - set = async (obj) => { - const key = this.getKey(obj); - const value = this.jc.encode(obj); - await this.kv.put(key, value); + delete = (obj) => { + if (this.dkv == null) throw Error("closed"); + this.dkv.delete(this.getKey(obj)); }; - delete = async (obj) => { - await this.kv.delete(this.getKey(obj)); - }; - - private decode = (mesg) => { - return mesg?.sm?.data != null ? this.jc.decode(mesg.sm.data) : null; - }; - - get = async (obj?, options: { natsKeys?: boolean } = {}) => { + get = (obj?) => { + if (this.dkv == null) throw Error("closed"); if (obj == null) { - const { all: raw } = await getAllFromKv({ - kv: this.kv, - key: `${this.natsKeyPrefix}.>`, - }); - if (options.natsKeys) { - // gets everything as a map with NATS keys but decoded values. - // This is used by the database changefeed stuff. - for (const key in raw) { - raw[key] = this.jc.decode(raw[key]); - } - return raw; - } - const all: any = {}; - for (const x of Object.values(raw)) { - const value = this.jc.decode(x); - all[this.primaryString(value)] = value; - } - return all; - } else { - return this.decode(await this.kv.get(this.getKey(obj))); + return this.dkv.get(); } + return this.dkv.get(this.getKey(obj)); }; - // watch for new changes - async *watch() { - if (this.kv == null) { - throw Error("not initialized"); - } - const w = await this.kv.watch({ - key: `${this.natsKeyPrefix}.>`, - include: "updates", - }); - this.watches.push(w); - for await (const { value } of w) { - if (this.state == "closed") { - return; - } - yield this.jc.decode(value); - } - } - close = () => { + if (this.state == "closed") return; this.set_state("closed"); this.removeAllListeners(); - for (const w of this.watches) { - w.stop(); - } - this.watches = []; + this.dkv?.close(); + delete this.dkv; + // @ts-ignore + delete this.env; }; } diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index bcf0a05747..92781448f3 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -1,3 +1,5 @@ +import jsonStableStringify from "json-stable-stringify"; + // Get the number of NON-deleted keys in a nats kv store, matching a given subject: export async function numKeys(kv, x: string | string[] = ">"): Promise { let num = 0; @@ -109,3 +111,13 @@ export function nanos(millis: number): Nanos { export function millis(ns: Nanos): number { return Math.floor(ns / 1000000); } + +export function toKey(x): string | undefined { + if (x === undefined) { + return undefined; + } else if (typeof x === "object") { + return jsonStableStringify(x); + } else { + return `${x}`; + } +} diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 46957d0d5d..e4be78f8d5 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -348,14 +348,14 @@ function projectSubjects(project_id: string) { pub.add(`*.project-${project_id}.>`); sub.add(`*.project-${project_id}.>`); + // The unique project-wide jetstream key:value store pub.add(`$JS.*.*.*.KV_project-${project_id}`); pub.add(`$JS.*.*.*.KV_project-${project_id}.>`); - for (const name of ["", "-patches"]) { - pub.add(`$JS.*.*.*.project-${project_id}${name}`); - pub.add(`$JS.*.*.*.project-${project_id}${name}.>`); - pub.add(`$JS.*.*.*.*.project-${project_id}${name}.>`); - } + // The unique project-wide jetstream stream: + pub.add(`$JS.*.*.*.project-${project_id}`); + pub.add(`$JS.*.*.*.project-${project_id}.>`); + pub.add(`$JS.*.*.*.*.project-${project_id}.>`); return { pub, sub }; } diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index c540fd4e27..f3f899c257 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -13,7 +13,6 @@ export class NatsChangefeed extends EventEmitter { private options; private state: State = "disconnected"; private natsSynctable?; - private watch?; constructor({ client, query, options }: { client; query; options? }) { super(); @@ -29,13 +28,13 @@ export class NatsChangefeed extends EventEmitter { this.natsSynctable = await this.client.nats_client.changefeed(this.query); this.interest(); this.startWatch(); - return Object.values(await this.natsSynctable.get()); + return Object.values(this.natsSynctable.get()); }; close = (): void => { this.natsSynctable?.close(); this.state = "closed"; - this.emit("close"); + this.emit("close"); // yes "close" not "closed" ;-( }; get_state = (): string => { @@ -51,13 +50,12 @@ export class NatsChangefeed extends EventEmitter { } }; - private startWatch = async () => { + private startWatch = () => { if (this.natsSynctable == null) { return; } - this.watch = await this.natsSynctable.watch(); - for await (const new_val of this.watch) { - this.emit("update", { action: "update", new_val }); - } + this.natsSynctable.on("change", ({ value: new_val, prev: old_val }) => { + this.emit("update", { action: "update", new_val, old_val }); + }); }; } From c903fa1f721fcba50c71e996b039951ef4641d6c Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 10 Feb 2025 01:49:22 +0000 Subject: [PATCH 161/281] nats changefeeds -- fix some bugs with rewrite; this makes sense finally and works well --- src/packages/database/nats/changefeeds.ts | 38 ++++++++++++++----- src/packages/nats/sync/synctable-kv-atomic.ts | 16 +++++--- .../sync/editor/string/test/client-test.ts | 6 +-- src/packages/sync/table/changefeed-nats.ts | 3 +- 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/packages/database/nats/changefeeds.ts b/src/packages/database/nats/changefeeds.ts index b0e118a636..67ed01fb19 100644 --- a/src/packages/database/nats/changefeeds.ts +++ b/src/packages/database/nats/changefeeds.ts @@ -8,6 +8,8 @@ require("@cocalc/database/nats/changefeeds").init() + echo 'require("@cocalc/database/nats/changefeeds").init()' | node + */ import getLogger from "@cocalc/backend/logger"; @@ -26,14 +28,13 @@ import jsonStableStringify from "json-stable-stringify"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { uuid } from "@cocalc/util/misc"; import { delay } from "awaiting"; -import { Svcm, type ServiceMsg } from "@nats-io/services"; -import { type QueuedIterator } from "nats"; +import { Svcm } from "@nats-io/services"; const logger = getLogger("database:nats:changefeeds"); const jc = JSONCodec(); -let api: QueuedIterator | null = null; +let api: any | null = null; export async function init() { const subject = "hub.*.*.db"; logger.debug(`init -- subject='${subject}', options=`, { @@ -50,7 +51,7 @@ export async function init() { description: "CoCalc Database Service (changefeeds)", }); - const api = service.addEndpoint("api", { subject }); + api = service.addEndpoint("api", { subject }); for await (const mesg of api) { handleRequest(mesg, nc); @@ -58,8 +59,9 @@ export async function init() { } export function terminate() { - logger.debug("terminating"); + logger.debug("terminating service"); api?.stop(); + api = null; // also, stop reporting data into the streams cancelAllChangefeeds(); } @@ -133,14 +135,21 @@ function cancelAllChangefeeds() { const createChangefeed = reuseInFlight( async (opts, nc) => { const query = opts.query; - const hash = sha1(jsonStableStringify(query)); + // the query *AND* the user making it define the thing: + const user = { account_id: opts.account_id, project_id: opts.project_id }; + const hash = sha1( + jsonStableStringify({ + query, + ...user, + }), + ); const now = Date.now(); if (changefeedInterest[hash]) { changefeedInterest[hash] = now; - logger.debug("using existing changefeed for", queryTable(query)); + logger.debug("using existing changefeed for", queryTable(query), user); return; } - logger.debug("creating new changefeed for", queryTable(query)); + logger.debug("creating new changefeed for", queryTable(query), user); const changes = uuid(); changefeedHashes[changes] = hash; const env = { nc, jc, sha1 }; @@ -151,6 +160,7 @@ const createChangefeed = reuseInFlight( project_id: opts.project_id, atomic: true, }); + await synctable.init(); // if (global.z == null) { // global.z = {}; @@ -162,9 +172,17 @@ const createChangefeed = reuseInFlight( cb(err ?? "missing result"); return; } + const current = synctable.get(); + const databaseKeys = new Set(); for (const obj of rows) { + databaseKeys.add(synctable.getKey(obj)); synctable.set(obj); } + for (const key in current) { + if (!databaseKeys.has(key)) { + synctable.delete(key); + } + } cb(); }; @@ -177,7 +195,9 @@ const createChangefeed = reuseInFlight( return; } if (action == "insert" || action == "update") { - synctable.set(new_val); + const cur = synctable.get(new_val); + // logger.debug({ table: queryTable(query), action, new_val, old_val }); + synctable.set({ ...cur, ...new_val }); } else if (action == "delete") { synctable.delete(old_val); } else if (action == "close") { diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index f1f4a64b3b..39824127a1 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -65,7 +65,11 @@ export class SyncTableKVAtomic extends EventEmitter { this.set_state("connected"); }; - getKey = (obj): string => { + getKey = (obj_or_key): string => { + if (typeof obj_or_key == "string") { + return obj_or_key; + } + const obj = obj_or_key; if (this.primaryKeys.length === 1) { return toKey(obj[this.primaryKeys[0]] ?? "")!; } else { @@ -79,17 +83,17 @@ export class SyncTableKVAtomic extends EventEmitter { this.dkv.set(this.getKey(obj), obj); }; - delete = (obj) => { + delete = (obj_or_key) => { if (this.dkv == null) throw Error("closed"); - this.dkv.delete(this.getKey(obj)); + this.dkv.delete(this.getKey(obj_or_key)); }; - get = (obj?) => { + get = (obj_or_key?) => { if (this.dkv == null) throw Error("closed"); - if (obj == null) { + if (obj_or_key == null) { return this.dkv.get(); } - return this.dkv.get(this.getKey(obj)); + return this.dkv.get(this.getKey(obj_or_key)); }; close = () => { diff --git a/src/packages/sync/editor/string/test/client-test.ts b/src/packages/sync/editor/string/test/client-test.ts index 798f3a8d5f..4fa52d41b1 100644 --- a/src/packages/sync/editor/string/test/client-test.ts +++ b/src/packages/sync/editor/string/test/client-test.ts @@ -168,14 +168,14 @@ export class Client extends EventEmitter implements Client0 { _options: any, _throttle_changes?: number, ): Promise { - throw Error("not implemented"); + throw Error("synctable_database: not implemented"); } async synctable_nats(_query: any): Promise { - throw Error("not implemented"); + throw Error("synctable_nats: not implemented"); } async pubsub_nats(_query: any): Promise { - throw Error("not implemented"); + throw Error("pubsub_nats: not implemented"); } // account_id or project_id diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index f3f899c257..36987d3b79 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -28,7 +28,8 @@ export class NatsChangefeed extends EventEmitter { this.natsSynctable = await this.client.nats_client.changefeed(this.query); this.interest(); this.startWatch(); - return Object.values(this.natsSynctable.get()); + const v = this.natsSynctable.get(); + return Object.values(v); }; close = (): void => { From 0874950321ef31f4cef6892e05d210f190e2ee63 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 10 Feb 2025 05:13:15 +0000 Subject: [PATCH 162/281] nats: implemented new "dko" = distributed key:object store, where when object changes we only send over nats the *changed* key/value, rather than entire object. - this seems to mostly work when I enabled it. - I don't know if this is a good idea or not yet... --- src/packages/backend/nats/sync.ts | 15 +- src/packages/frontend/nats/client.ts | 12 +- src/packages/nats/sync/dko.ts | 213 ++++++++++++++++++ src/packages/nats/sync/dkv.ts | 19 +- src/packages/nats/sync/general-dkv.ts | 16 +- src/packages/nats/sync/synctable-kv-atomic.ts | 56 ++++- src/packages/nats/sync/synctable-kv.ts | 2 + src/packages/nats/sync/synctable.ts | 20 +- src/packages/project/nats/sync.ts | 5 + src/packages/sync/table/changefeed-nats.ts | 4 +- 10 files changed, 328 insertions(+), 34 deletions(-) create mode 100644 src/packages/nats/sync/dko.ts diff --git a/src/packages/backend/nats/sync.ts b/src/packages/backend/nats/sync.ts index c59a632752..5c346aa78b 100644 --- a/src/packages/backend/nats/sync.ts +++ b/src/packages/backend/nats/sync.ts @@ -5,22 +5,27 @@ import { } from "@cocalc/nats/sync/dstream"; import { kv as createKV, type KV } from "@cocalc/nats/sync/kv"; import { dkv as createDKV, type DKV } from "@cocalc/nats/sync/dkv"; +import { dko as createDKO, type DKO } from "@cocalc/nats/sync/dko"; import { getEnv } from "@cocalc/backend/nats/env"; -export type { Stream, DStream, KV, DKV }; +export type { Stream, DStream, KV, DKV, DKO }; -export async function stream(opts) { +export async function stream(opts): Promise { return await createStream({ env: await getEnv(), ...opts }); } -export async function dstream(opts) { +export async function dstream(opts): Promise { return await createDstream({ env: await getEnv(), ...opts }); } -export async function kv(opts) { +export async function kv(opts): Promise { return await createKV({ env: await getEnv(), ...opts }); } -export async function dkv(opts) { +export async function dkv(opts): Promise { return await createDKV({ env: await getEnv(), ...opts }); } + +export async function dko(opts): Promise { + return await createDKO({ env: await getEnv(), ...opts }); +} diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 18534deb08..4e56eb0666 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -20,6 +20,7 @@ import { PubSub } from "@cocalc/nats/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; import { kv, type KVOptions } from "@cocalc/nats/sync/kv"; import { dkv, type DKVOptions } from "@cocalc/nats/sync/dkv"; +import { dko, type DKOOptions } from "@cocalc/nats/sync/dko"; import { stream, type UserStreamOptions } from "@cocalc/nats/sync/stream"; import { dstream } from "@cocalc/nats/sync/dstream"; import { initApi } from "@cocalc/frontend/nats/api"; @@ -316,9 +317,9 @@ export class NatsClient { } }; - changefeed = async (query) => { + changefeed = async (query, { atomic = true }: { atomic?: boolean } = {}) => { this.changefeedInterest(query, true); - return await this.synctable(query, { atomic: true }); + return await this.synctable(query, { atomic }); }; // DEPRECATED @@ -414,6 +415,13 @@ export class NatsClient { return await dkv({ env: await this.getEnv(), ...opts }); }; + dko = async (opts: Partial) => { + // if (!opts.account_id && !opts.project_id && opts.limits != null) { + // throw Error("account client can't set limits on public stream"); + // } + return await dko({ env: await this.getEnv(), ...opts }); + }; + microservicesClient = async () => { const nc = await this.getConnection(); // @ts-ignore diff --git a/src/packages/nats/sync/dko.ts b/src/packages/nats/sync/dko.ts new file mode 100644 index 0000000000..0509a7556d --- /dev/null +++ b/src/packages/nats/sync/dko.ts @@ -0,0 +1,213 @@ +/* +Distributed eventually consistent key:object store, where changes propogate sparsely. + +The "values" MUST be objects and no keys or fields of objects can container the sep character, +which is '|' by default. + +DEVELOPMENT: + +~/cocalc/src/packages/backend n +Welcome to Node.js v18.17.1. +Type ".help" for more information. +> t = await require("@cocalc/backend/nats/sync").dko({name:'test'}) + +*/ + +import { EventEmitter } from "events"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { dkv as createDKV, DKV, DKVOptions } from "./dkv"; +import { userKvKey } from "./kv"; +import { is_object } from "@cocalc/util/misc"; + +export interface DKOOptions extends DKVOptions { + sep?: string; +} + +export class DKO extends EventEmitter { + opts: DKOOptions; + sep: string; + dkv?: DKV; + + constructor(opts: DKOOptions) { + super(); + this.opts = opts; + this.sep = opts.sep ?? "|"; + this.init(); + return new Proxy(this, { + deleteProperty(target, prop) { + target.delete(prop); + return true; + }, + set(target, prop, value) { + prop = String(prop); + if (prop == "_eventsCount") { + target[prop] = value; + return true; + } + if (target[prop] != null) { + throw Error(`method name '${prop}' is read only`); + } + target.set(prop, value); + return true; + }, + get(target, prop) { + return target[String(prop)] ?? target.get(String(prop)); + }, + }); + } + + init = reuseInFlight(async () => { + if (this.dkv != null) { + throw Error("already initialized"); + } + this.dkv = await createDKV({ + ...this.opts, + name: dkoPrefix(this.opts.name), + }); + this.dkv.on("change", ({ key: path, value }) => { + const { key, field } = this.fromPath(path); + if (!field) { + this.emit("change", { key }); + } else { + this.emit("change", { key, field, value }); + } + }); + this.dkv.on("reject", ({ key: path, value }) => { + const { key, field } = this.fromPath(path); + if (!field) { + this.emit("reject", { key }); + } else { + this.emit("reject", { key, field, value }); + } + }); + await this.dkv.init(); + }); + + close = () => { + if (this.dkv == null) { + return; + } + this.dkv.close(); + delete this.dkv; + this.emit("closed"); + this.removeAllListeners(); + }; + + private toPath = (key: string, field: string): string => { + return `${key}${this.sep}${field}`; + }; + + private fromPath = (path: string): { key: string; field: string } => { + const [key, field] = path.split(this.sep); + return { key, field }; + }; + + delete = (key) => { + if (this.dkv == null) { + throw Error("closed"); + } + const fields = this.dkv.get(key); + if (fields == null) { + return; + } + for (const field of fields) { + this.dkv.delete(this.toPath(key, field)); + } + }; + + get = (key?) => { + if (this.dkv == null) { + throw Error("closed"); + } + if (key == null) { + // get everything + const all = this.dkv.get(); + const result: any = {}; + for (const x in all) { + const { key, field } = this.fromPath(x); + if (!field) { + continue; + } + if (result[key] == null) { + result[key] = { [field]: all[x] }; + } else { + result[key][field] = all[x]; + } + } + return result; + } else { + const fields = this.dkv.get(key); + if (fields == null) { + return undefined; + } + const x: any = {}; + for (const field of fields) { + x[field] = this.dkv.get(this.toPath(key, field)); + } + return x; + } + }; + + set = (key: string, obj: any) => { + if (this.dkv == null) { + throw Error("closed"); + } + if (obj == null) { + this.delete(key); + return; + } + if (!is_object(obj)) { + throw Error("values must be objects"); + } + const fields = Object.keys(obj); + this.dkv.set(key, fields); + for (const field of fields) { + this.dkv.set(this.toPath(key, field), obj[field]); + } + }; + + hasUnsavedChanges = () => { + return !!this.dkv?.hasUnsavedChanges(); + }; + + unsavedChanges = () => { + const dkv = this.dkv; + if (dkv == null) { + return []; + } + const v = dkv.unsavedChanges(); + const w: { key: string; field: string }[] = []; + for (const path of v) { + const { key, field } = this.fromPath(path); + if (field) { + w.push({ key, field }); + } + } + return w; + }; + + save = async () => { + await this.dkv?.save(); + }; +} + +const cache: { [key: string]: DKO } = {}; +export const dko = reuseInFlight( + async (opts: DKOOptions) => { + const key = userKvKey(opts); + if (cache[key] == null) { + const k = new DKO(opts); + await k.init(); + k.on("closed", () => delete cache[key]); + cache[key] = k; + } + return cache[key]!; + }, + { + createKey: (args) => userKvKey(args[0]), + }, +); + +function dkoPrefix(name: string): string { + return `__dko__${name}`; +} diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index 938d2a02f1..ed2adacbc0 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -4,7 +4,7 @@ Always Consistent Centralized Key Value Store DEVELOPMENT: -~/cocalc/src/packages/backend n +~/cocalc/src/packages/backend$ n Welcome to Node.js v18.17.1. Type ".help" for more information. > t = await require("@cocalc/backend/nats/sync").dkv({name:'test'}) @@ -27,6 +27,7 @@ export class DKV extends EventEmitter { name: string; private prefix: string; private sha1; + private opts; constructor({ name, @@ -45,13 +46,14 @@ export class DKV extends EventEmitter { this.name = name; this.sha1 = env.sha1 ?? sha1; this.prefix = this.sha1(name); - this.generalDKV = new GeneralDKV({ + this.opts = { name: kvname, filter: `${this.prefix}.>`, env, merge, limits, - }); + }; + this.init(); return new Proxy(this, { deleteProperty(target, prop) { @@ -77,9 +79,10 @@ export class DKV extends EventEmitter { } init = reuseInFlight(async () => { - if (this.generalDKV == null) { - throw Error("closed"); + if (this.generalDKV != null) { + return; } + this.generalDKV = new GeneralDKV(this.opts); this.generalDKV.on("change", ({ value, prev }) => { if (value != null) { this.emit("change", { key: value.key, value: value.value }); @@ -102,6 +105,8 @@ export class DKV extends EventEmitter { } this.generalDKV.close(); delete this.generalDKV; + // @ts-ignore + delete this.opts; this.emit("closed"); this.removeAllListeners(); }; @@ -171,6 +176,10 @@ export class DKV extends EventEmitter { } return generalDKV.unsavedChanges().map((key) => generalDKV.get(key)?.key); }; + + save = async () => { + await this.generalDKV?.save(); + }; } const cache: { [key: string]: DKV } = {}; diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index 7cd786f802..6cc18b713a 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -158,13 +158,13 @@ export class GeneralDKV extends EventEmitter { } else { try { value = this.merge?.({ key, local, remote, prev }) ?? local; -// console.log("handle merge conflict", { -// key, -// local, -// remote, -// prev, -// value, -// }); + // console.log("handle merge conflict", { + // key, + // local, + // remote, + // prev, + // value, + // }); } catch { // user provided a merge function that throws an exception. We select local, since // it is the newest, i.e., "last write wins" @@ -251,7 +251,7 @@ export class GeneralDKV extends EventEmitter { return Object.keys(this.local); }; - private save = reuseInFlight(async () => { + save = reuseInFlight(async () => { let d = 100; while (true) { try { diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index 39824127a1..4ca4621aa1 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -10,17 +10,20 @@ import { client_db } from "@cocalc/util/db-schema/client-db"; import type { NatsEnv, State } from "@cocalc/nats/types"; import { EventEmitter } from "events"; import { dkv as createDkv, type DKV } from "./dkv"; +import { dko as createDko, type DKO } from "./dko"; import jsonStableStringify from "json-stable-stringify"; import { toKey } from "@cocalc/nats/util"; +import { wait } from "@cocalc/util/async-wait"; export class SyncTableKVAtomic extends EventEmitter { public readonly table; private query; + private atomic: boolean; private primaryKeys: string[]; private project_id?: string; private account_id?: string; private state: State = "disconnected"; - private dkv?: DKV; + private dkv?: DKV | DKO; private env; constructor({ @@ -28,13 +31,16 @@ export class SyncTableKVAtomic extends EventEmitter { env, account_id, project_id, + atomic, }: { query; env: NatsEnv; account_id?: string; project_id?: string; + atomic?: boolean; }) { super(); + this.atomic = !!atomic; this.query = query; this.env = env; this.table = keys(query)[0]; @@ -53,13 +59,25 @@ export class SyncTableKVAtomic extends EventEmitter { }; init = async () => { - this.dkv = await createDkv({ - name: jsonStableStringify(this.query), - account_id: this.account_id, - project_id: this.project_id, - env: this.env, - }); + if (this.atomic) { + this.dkv = await createDkv({ + name: jsonStableStringify(this.query), + account_id: this.account_id, + project_id: this.project_id, + env: this.env, + }); + } else { + this.dkv = await createDko({ + name: jsonStableStringify(this.query), + account_id: this.account_id, + project_id: this.project_id, + env: this.env, + }); + } this.dkv.on("change", (x) => { + if (!this.atomic) { + x = { ...x, value: this.dkv?.get(x.key) }; + } this.emit("change", x); }); this.set_state("connected"); @@ -96,6 +114,18 @@ export class SyncTableKVAtomic extends EventEmitter { return this.dkv.get(this.getKey(obj_or_key)); }; + get_one = () => { + if (this.dkv == null) throw Error("closed"); + // TODO: insanely inefficient, especially if !atomic! + for (const key in this.dkv.get()) { + return this.get(key); + } + }; + + save = async () => { + await this.dkv?.save(); + }; + close = () => { if (this.state == "closed") return; this.set_state("closed"); @@ -105,4 +135,16 @@ export class SyncTableKVAtomic extends EventEmitter { // @ts-ignore delete this.env; }; + + public async wait(until: Function, timeout: number = 30): Promise { + if (this.state == "closed") { + throw Error("wait: must not be closed"); + } + return await wait({ + obj: this, + until, + timeout, + change_event: "change", + }); + } } diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index f134ac3bf3..3271895bac 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -1,4 +1,6 @@ /* +DEPRECATED + Nats implementation of the idea of a "SyncTable". This is ONLY for synctables in the scope of a single project, e.g., diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/nats/sync/synctable.ts index 580735743e..028988c4df 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/nats/sync/synctable.ts @@ -27,8 +27,8 @@ export function createSyncTable({ immutable?: boolean; // if true for SyncTableKVAtomic, then get/get_one output immutable.js objects }) { if (stream) { - if (!atomic) { - throw Error("non-atomic stream not implemented yet"); + if (atomic === false) { + throw Error("streams must be atomic"); } return new SyncTableStream({ query, @@ -37,16 +37,24 @@ export function createSyncTable({ project_id, ...options, }); - } - if (atomic) { + } else { + if (options?.immutable !== undefined) { + // for now if immutable specified at all, do this (for now so project save works) + return new SyncTableKV({ + query, + env, + account_id, + project_id, + ...options, + }); + } return new SyncTableKVAtomic({ query, env, account_id, project_id, + atomic, ...options, }); - } else { - return new SyncTableKV({ query, env, account_id, project_id, ...options }); } } diff --git a/src/packages/project/nats/sync.ts b/src/packages/project/nats/sync.ts index c06675edbb..e47413bcf2 100644 --- a/src/packages/project/nats/sync.ts +++ b/src/packages/project/nats/sync.ts @@ -5,6 +5,7 @@ import { } from "@cocalc/nats/sync/dstream"; import { kv as createKV, type KV } from "@cocalc/nats/sync/kv"; import { dkv as createDKV, type DKV } from "@cocalc/nats/sync/dkv"; +import { dko as createDKO, type DKO } from "@cocalc/nats/sync/dko"; import { getEnv } from "./env"; import { project_id } from "@cocalc/project/data"; @@ -25,3 +26,7 @@ export async function kv(opts): Promise { export async function dkv(opts): Promise { return await createDKV({ project_id, env: await getEnv(), ...opts }); } + +export async function dko(opts): Promise { + return await createDKO({ project_id, env: await getEnv(), ...opts }); +} diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index 36987d3b79..f23cd34432 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -25,7 +25,9 @@ export class NatsChangefeed extends EventEmitter { } connect = async () => { - this.natsSynctable = await this.client.nats_client.changefeed(this.query); + this.natsSynctable = await this.client.nats_client.changefeed(this.query, { + atomic: true, + }); this.interest(); this.startWatch(); const v = this.natsSynctable.get(); From 4339bb832970505f9ba41418ff61a9c11fd0af09 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 10 Feb 2025 15:24:44 +0000 Subject: [PATCH 163/281] switch to using new synctable based on new dko. --- src/packages/database/nats/changefeeds.ts | 3 +- src/packages/frontend/nats/client.ts | 7 +-- src/packages/nats/sync/dko.ts | 8 +++ src/packages/nats/sync/synctable-kv-atomic.ts | 46 ++++++++++++++--- src/packages/nats/sync/synctable.ts | 17 +++---- src/packages/project/nats/open-files.ts | 24 +++------ src/packages/project/nats/synctable.ts | 51 +++++++++---------- src/packages/sync/table/changefeed-nats.ts | 3 +- 8 files changed, 94 insertions(+), 65 deletions(-) diff --git a/src/packages/database/nats/changefeeds.ts b/src/packages/database/nats/changefeeds.ts index 67ed01fb19..fbbe2f98f1 100644 --- a/src/packages/database/nats/changefeeds.ts +++ b/src/packages/database/nats/changefeeds.ts @@ -158,7 +158,8 @@ const createChangefeed = reuseInFlight( env, account_id: opts.account_id, project_id: opts.project_id, - atomic: true, + atomic: false, + immutable: false, }); await synctable.init(); diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 4e56eb0666..7859e4823e 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -256,6 +256,7 @@ export class NatsClient { options?: { obj?: object; atomic?: boolean; + immutable?: boolean; stream?: boolean; pubsub?: boolean; throttleChanges?: number; @@ -264,7 +265,7 @@ export class NatsClient { }, ): Promise => { query = parse_query(query); - const key = JSON.stringify(query); + const key = JSON.stringify({ query, options }); if (this.synctableCache[key] != null) { return this.synctableCache[key]; } @@ -317,9 +318,9 @@ export class NatsClient { } }; - changefeed = async (query, { atomic = true }: { atomic?: boolean } = {}) => { + changefeed = async (query, options?) => { this.changefeedInterest(query, true); - return await this.synctable(query, { atomic }); + return await this.synctable(query, options); }; // DEPRECATED diff --git a/src/packages/nats/sync/dko.ts b/src/packages/nats/sync/dko.ts index 0509a7556d..d8ee62e2b0 100644 --- a/src/packages/nats/sync/dko.ts +++ b/src/packages/nats/sync/dko.ts @@ -65,6 +65,10 @@ export class DKO extends EventEmitter { name: dkoPrefix(this.opts.name), }); this.dkv.on("change", ({ key: path, value }) => { + if (path == null) { + // TODO: why would this happen? + return; + } const { key, field } = this.fromPath(path); if (!field) { this.emit("change", { key }); @@ -73,6 +77,10 @@ export class DKO extends EventEmitter { } }); this.dkv.on("reject", ({ key: path, value }) => { + if (path == null) { + // TODO: would this happen? + return; + } const { key, field } = this.fromPath(path); if (!field) { this.emit("reject", { key }); diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts index 4ca4621aa1..6219555600 100644 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ b/src/packages/nats/sync/synctable-kv-atomic.ts @@ -14,6 +14,7 @@ import { dko as createDko, type DKO } from "./dko"; import jsonStableStringify from "json-stable-stringify"; import { toKey } from "@cocalc/nats/util"; import { wait } from "@cocalc/util/async-wait"; +import { fromJS, Map } from "immutable"; export class SyncTableKVAtomic extends EventEmitter { public readonly table; @@ -25,6 +26,7 @@ export class SyncTableKVAtomic extends EventEmitter { private state: State = "disconnected"; private dkv?: DKV | DKO; private env; + private getHook: Function; constructor({ query, @@ -32,20 +34,27 @@ export class SyncTableKVAtomic extends EventEmitter { account_id, project_id, atomic, + immutable, }: { query; env: NatsEnv; account_id?: string; project_id?: string; atomic?: boolean; + immutable?: boolean; }) { super(); this.atomic = !!atomic; + this.getHook = immutable ? fromJS : (x) => x; this.query = query; this.env = env; this.table = keys(query)[0]; - this.account_id = account_id ?? query[this.table][0].account_id; - this.project_id = project_id; + if (query[this.table][0].string_id && query[this.table][0].project_id) { + this.project_id = query[this.table][0].project_id; + } else { + this.account_id = account_id ?? query[this.table][0].account_id; + this.project_id = project_id; + } this.primaryKeys = client_db.primary_keys(this.table); } @@ -58,17 +67,34 @@ export class SyncTableKVAtomic extends EventEmitter { return this.state; }; + private getName = () => { + const primary: any = {}; + const spec = this.query[this.table][0]; + for (const key of this.primaryKeys) { + const val = spec[key]; + if (val != null) { + primary[key] = val; + } + } + if (Object.keys(primary).length == 0) { + return this.table; + } else { + return `${this.table}-${jsonStableStringify(primary)}`; + } + }; + init = async () => { + const name = this.getName(); if (this.atomic) { this.dkv = await createDkv({ - name: jsonStableStringify(this.query), + name, account_id: this.account_id, project_id: this.project_id, env: this.env, }); } else { this.dkv = await createDko({ - name: jsonStableStringify(this.query), + name, account_id: this.account_id, project_id: this.project_id, env: this.env, @@ -87,7 +113,10 @@ export class SyncTableKVAtomic extends EventEmitter { if (typeof obj_or_key == "string") { return obj_or_key; } - const obj = obj_or_key; + let obj = obj_or_key; + if (Map.isMap(obj)) { + obj = obj.toJS(); + } if (this.primaryKeys.length === 1) { return toKey(obj[this.primaryKeys[0]] ?? "")!; } else { @@ -98,6 +127,9 @@ export class SyncTableKVAtomic extends EventEmitter { set = (obj) => { if (this.dkv == null) throw Error("closed"); + if (Map.isMap(obj)) { + obj = obj.toJS(); + } this.dkv.set(this.getKey(obj), obj); }; @@ -109,9 +141,9 @@ export class SyncTableKVAtomic extends EventEmitter { get = (obj_or_key?) => { if (this.dkv == null) throw Error("closed"); if (obj_or_key == null) { - return this.dkv.get(); + return this.getHook(this.dkv.get()); } - return this.dkv.get(this.getKey(obj_or_key)); + return this.getHook(this.dkv.get(this.getKey(obj_or_key))); }; get_one = () => { diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/nats/sync/synctable.ts index 028988c4df..20541d7cc4 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/nats/sync/synctable.ts @@ -16,6 +16,7 @@ export function createSyncTable({ project_id, atomic, stream, + immutable, ...options }: { query; @@ -24,12 +25,15 @@ export function createSyncTable({ project_id?: string; atomic?: boolean; stream?: boolean; - immutable?: boolean; // if true for SyncTableKVAtomic, then get/get_one output immutable.js objects + immutable?: boolean; // if true, then get/set works with immutable.js objects instead. }) { if (stream) { if (atomic === false) { throw Error("streams must be atomic"); } + if (immutable) { + throw Error("immutable not yet supported for streams"); + } return new SyncTableStream({ query, env, @@ -38,22 +42,13 @@ export function createSyncTable({ ...options, }); } else { - if (options?.immutable !== undefined) { - // for now if immutable specified at all, do this (for now so project save works) - return new SyncTableKV({ - query, - env, - account_id, - project_id, - ...options, - }); - } return new SyncTableKVAtomic({ query, env, account_id, project_id, atomic, + immutable, ...options, }); } diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 3da4b862a2..4598f658d9 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -41,20 +41,12 @@ export async function init() { project_id, env: await getEnv(), }); - runLoop(); -} - -function runLoop() { - logger.debug("starting run loop"); - if (openFiles != null) { - const entries: { [path: string]: Entry } = {}; - closeIgnoredFiles(entries, openFiles); - openFiles.on("change", (entry) => { - entries[entry.path] = entry; - handleChange(entry); - }); - } - logger.debug("exiting open files run loop"); + const entries: { [path: string]: Entry } = {}; + closeIgnoredFiles(entries, openFiles); + openFiles.on("change", (entry) => { + entries[entry.path] = entry; + handleChange(entry); + }); } export function terminate() { @@ -167,12 +159,11 @@ const openSyncDoc = reuseInFlight(async (path: string) => { ); x = await getTypeAndOpts(syncstrings); } catch (err) { - logger.debug(`openSyncDoc failed ${err}`); + logger.debug(`openSyncDoc failed - error = ${err}`); return; } const { type, opts } = x; logger.debug("openSyncDoc got", { path, type, opts }); - console.log("openSyncDoc got", { path, type, opts }); let doc; if (type == "string") { @@ -197,6 +188,7 @@ const openSyncDoc = reuseInFlight(async (path: string) => { async function getTypeAndOpts( syncstrings, ): Promise<{ type: string; opts: any }> { + global.z = {syncstrings} let s = syncstrings.get_one(); if (s == null) { await syncstrings.wait(() => { diff --git a/src/packages/project/nats/synctable.ts b/src/packages/project/nats/synctable.ts index 51afd8412c..9a9f77b691 100644 --- a/src/packages/project/nats/synctable.ts +++ b/src/packages/project/nats/synctable.ts @@ -10,33 +10,32 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; const jc = JSONCodec(); const cache: { [key: string]: SyncTable } = {}; -const synctable = reuseInFlight( - async (query, options: { obj?; atomic?: boolean; stream?: boolean }) => { - const key = JSON.stringify(query); - if (cache[key] == null) { - const nc = await getConnection(); - query = parse_query(query); - const table = keys(query)[0]; - const obj = options?.obj; - if (obj != null) { - for (const k in obj) { - query[table][0][k] = obj[k]; - } +const synctable = reuseInFlight(async (query, options?) => { + const key = JSON.stringify(query); + if (cache[key] == null) { + const nc = await getConnection(); + query = parse_query(query); + const table = keys(query)[0]; + const obj = options?.obj; + if (obj != null) { + for (const k in obj) { + query[table][0][k] = obj[k]; } - query[table][0].project_id = project_id; - const s = createSyncTable({ - ...options, - query, - env: { sha1, jc, nc }, - }); - await s.init(); - cache[key] = s; - s.on("closed", () => { - delete cache[key]; - }); } - return cache[key]; - }, -); + query[table][0].project_id = project_id; + const s = createSyncTable({ + project_id, + ...options, + query, + env: { sha1, jc, nc }, + }); + await s.init(); + cache[key] = s; + s.on("closed", () => { + delete cache[key]; + }); + } + return cache[key]; +}); export default synctable; diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index f23cd34432..e7c7c6a1f5 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -26,7 +26,8 @@ export class NatsChangefeed extends EventEmitter { connect = async () => { this.natsSynctable = await this.client.nats_client.changefeed(this.query, { - atomic: true, + atomic: false, + immutable: false, }); this.interest(); this.startWatch(); From 08b7302ad0c692c3a5a55099de123f9a80ff3a1a Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 10 Feb 2025 23:22:38 +0000 Subject: [PATCH 164/281] nats: fully switched everything to my new kv/stream implementations, which are robust to reconnects, etc. --- src/packages/frontend/app-framework/Table.ts | 2 +- src/packages/nats/sync/synctable-kv-atomic.ts | 182 ------- src/packages/nats/sync/synctable-kv.ts | 469 ++++-------------- src/packages/nats/sync/synctable-stream.ts | 17 +- src/packages/nats/sync/synctable.ts | 12 +- src/packages/project/nats/api/index.ts | 5 +- src/packages/sync/table/changefeed-nats.ts | 26 +- 7 files changed, 148 insertions(+), 565 deletions(-) delete mode 100644 src/packages/nats/sync/synctable-kv-atomic.ts diff --git a/src/packages/frontend/app-framework/Table.ts b/src/packages/frontend/app-framework/Table.ts index 306f17848a..caaeab2437 100644 --- a/src/packages/frontend/app-framework/Table.ts +++ b/src/packages/frontend/app-framework/Table.ts @@ -48,7 +48,7 @@ export abstract class Table { } this._table.on("error", (error) => { - console.warn(`Synctable error (table='${name}'): ${error}`); + console.warn(`Synctable error (table='${name}'):`, error); }); if (this._change != null) { diff --git a/src/packages/nats/sync/synctable-kv-atomic.ts b/src/packages/nats/sync/synctable-kv-atomic.ts deleted file mode 100644 index 6219555600..0000000000 --- a/src/packages/nats/sync/synctable-kv-atomic.ts +++ /dev/null @@ -1,182 +0,0 @@ -/* - - - - -*/ - -import { keys } from "lodash"; -import { client_db } from "@cocalc/util/db-schema/client-db"; -import type { NatsEnv, State } from "@cocalc/nats/types"; -import { EventEmitter } from "events"; -import { dkv as createDkv, type DKV } from "./dkv"; -import { dko as createDko, type DKO } from "./dko"; -import jsonStableStringify from "json-stable-stringify"; -import { toKey } from "@cocalc/nats/util"; -import { wait } from "@cocalc/util/async-wait"; -import { fromJS, Map } from "immutable"; - -export class SyncTableKVAtomic extends EventEmitter { - public readonly table; - private query; - private atomic: boolean; - private primaryKeys: string[]; - private project_id?: string; - private account_id?: string; - private state: State = "disconnected"; - private dkv?: DKV | DKO; - private env; - private getHook: Function; - - constructor({ - query, - env, - account_id, - project_id, - atomic, - immutable, - }: { - query; - env: NatsEnv; - account_id?: string; - project_id?: string; - atomic?: boolean; - immutable?: boolean; - }) { - super(); - this.atomic = !!atomic; - this.getHook = immutable ? fromJS : (x) => x; - this.query = query; - this.env = env; - this.table = keys(query)[0]; - if (query[this.table][0].string_id && query[this.table][0].project_id) { - this.project_id = query[this.table][0].project_id; - } else { - this.account_id = account_id ?? query[this.table][0].account_id; - this.project_id = project_id; - } - this.primaryKeys = client_db.primary_keys(this.table); - } - - private set_state = (state: State): void => { - this.state = state; - this.emit(state); - }; - - get_state = () => { - return this.state; - }; - - private getName = () => { - const primary: any = {}; - const spec = this.query[this.table][0]; - for (const key of this.primaryKeys) { - const val = spec[key]; - if (val != null) { - primary[key] = val; - } - } - if (Object.keys(primary).length == 0) { - return this.table; - } else { - return `${this.table}-${jsonStableStringify(primary)}`; - } - }; - - init = async () => { - const name = this.getName(); - if (this.atomic) { - this.dkv = await createDkv({ - name, - account_id: this.account_id, - project_id: this.project_id, - env: this.env, - }); - } else { - this.dkv = await createDko({ - name, - account_id: this.account_id, - project_id: this.project_id, - env: this.env, - }); - } - this.dkv.on("change", (x) => { - if (!this.atomic) { - x = { ...x, value: this.dkv?.get(x.key) }; - } - this.emit("change", x); - }); - this.set_state("connected"); - }; - - getKey = (obj_or_key): string => { - if (typeof obj_or_key == "string") { - return obj_or_key; - } - let obj = obj_or_key; - if (Map.isMap(obj)) { - obj = obj.toJS(); - } - if (this.primaryKeys.length === 1) { - return toKey(obj[this.primaryKeys[0]] ?? "")!; - } else { - // compound primary key - return toKey(this.primaryKeys.map((pk) => obj[pk]))!; - } - }; - - set = (obj) => { - if (this.dkv == null) throw Error("closed"); - if (Map.isMap(obj)) { - obj = obj.toJS(); - } - this.dkv.set(this.getKey(obj), obj); - }; - - delete = (obj_or_key) => { - if (this.dkv == null) throw Error("closed"); - this.dkv.delete(this.getKey(obj_or_key)); - }; - - get = (obj_or_key?) => { - if (this.dkv == null) throw Error("closed"); - if (obj_or_key == null) { - return this.getHook(this.dkv.get()); - } - return this.getHook(this.dkv.get(this.getKey(obj_or_key))); - }; - - get_one = () => { - if (this.dkv == null) throw Error("closed"); - // TODO: insanely inefficient, especially if !atomic! - for (const key in this.dkv.get()) { - return this.get(key); - } - }; - - save = async () => { - await this.dkv?.save(); - }; - - close = () => { - if (this.state == "closed") return; - this.set_state("closed"); - this.removeAllListeners(); - this.dkv?.close(); - delete this.dkv; - // @ts-ignore - delete this.env; - }; - - public async wait(until: Function, timeout: number = 30): Promise { - if (this.state == "closed") { - throw Error("wait: must not be closed"); - } - return await wait({ - obj: this, - until, - timeout, - change_event: "change", - }); - } -} diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 3271895bac..bf22305fc2 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -1,117 +1,31 @@ /* -DEPRECATED -Nats implementation of the idea of a "SyncTable". -This is ONLY for synctables in the scope of a single project, e.g., -syncstrings, listings, etc. -It uses a SINGLE NATS key-value store to represent -*all* SyncTables in a single project. + */ -import { Kvm } from "@nats-io/kv"; -import { sha1 } from "@cocalc/util/misc"; -import jsonStableStringify from "json-stable-stringify"; import { keys } from "lodash"; import { client_db } from "@cocalc/util/db-schema/client-db"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import type { NatsEnv, State } from "@cocalc/nats/types"; import { EventEmitter } from "events"; +import { dkv as createDkv, type DKV } from "./dkv"; +import { dko as createDko, type DKO } from "./dko"; +import jsonStableStringify from "json-stable-stringify"; +import { toKey } from "@cocalc/nats/util"; import { wait } from "@cocalc/util/async-wait"; -import { throttle } from "lodash"; import { fromJS, Map } from "immutable"; -import { getAllFromKv } from "@cocalc/nats/util"; -import { type NatsEnv } from "@cocalc/nats/types"; - -export function natsKeyPrefix({ - query, - atomic = false, - singleton, -}: { - query; - atomic?: boolean; - singleton?: string; -}) { - if (atomic) { - if (singleton) { - throw Error("not implemented"); - } - return sha1(jsonStableStringify({ query, atomic })); - } else { - // for non-atomic there's no problem with many different queries with the same primary keys. - // we thus just use the table's name - let prefix = keys(query)[0]; - if (singleton) { - prefix += "." + singleton; - } - return prefix; - } -} - -export async function getKv({ - nc, - project_id, - account_id, - options, -}: { - nc; - project_id?: string; - account_id?: string; - options?; -}) { - let name; - if (project_id) { - name = `project-${project_id}`; - } else if (account_id) { - name = `account-${account_id}`; - } else { - throw Error("one of account_id or project_id must be defined"); - } - const kvm = new Kvm(nc); - return await kvm.create(name, { compression: true, ...options }); -} - -export function toKey(x): string | undefined { - if (x === undefined) { - return undefined; - } else if (typeof x === "object") { - return jsonStableStringify(x); - } else { - return `${x}`; - } -} - -function isSingletonQuery(query) { - const table = keys(query)[0]; - const pattern = query[table][0]; - for (const key of client_db.primary_keys(table)) { - if (pattern[key] === null) { - // part of primary key is NOT specified, so not singleton - return false; - } - } - return true; -} export class SyncTableKV extends EventEmitter { - private kv?; - private nc; - private jc; - private sha1; public readonly table; - public readonly natsKeyPrefix; - private allFields: Set; + private query; + private atomic: boolean; private primaryKeys: string[]; - private primaryKeysSet: Set; - private fields: string[]; private project_id?: string; private account_id?: string; - private data: { [key: string]: any } = {}; - private state: "disconnected" | "connected" | "closed" = "disconnected"; - private updateListener?; - private changedKeys: Set = new Set(); - private specifiedByQuery: { [key: string]: any }; - private singleton?: string; + private state: State = "disconnected"; + private dkv?: DKV | DKO; + private env; private getHook: Function; constructor({ @@ -119,319 +33,150 @@ export class SyncTableKV extends EventEmitter { env, account_id, project_id, - throttleChanges = 100, + atomic, immutable, }: { query; env: NatsEnv; account_id?: string; project_id?: string; - throttleChanges?: number; + atomic?: boolean; immutable?: boolean; }) { super(); + this.atomic = !!atomic; this.getHook = immutable ? fromJS : (x) => x; - this.sha1 = env.sha1 ?? sha1; - this.nc = env.nc; - this.jc = env.jc; - this.throttledChangeEvent = throttle( - this.throttledChangeEvent, - throttleChanges, - { leading: false, trailing: true }, - ); - const table = keys(query)[0]; - this.table = table; - this.allFields = new Set(Object.keys(query[table][0])); - this.primaryKeys = client_db.primary_keys(table); - this.primaryKeysSet = new Set(this.primaryKeys); - this.project_id = project_id ?? query[table][0].project_id; - this.account_id = account_id ?? query[table][0].account_id; - this.singleton = isSingletonQuery(query) - ? this.natObjectKey(query[table][0]) - : undefined; - this.natsKeyPrefix = natsKeyPrefix({ - query, - atomic: false, - singleton: this.singleton, - }); - this.specifiedByQuery = {}; - for (const k in query[table][0]) { - const v = query[table][0][k]; - if (v != null) { - this.specifiedByQuery[k] = v; - } + this.query = query; + this.env = env; + this.table = keys(query)[0]; + if (query[this.table][0].string_id && query[this.table][0].project_id) { + this.project_id = query[this.table][0].project_id; + } else { + this.account_id = account_id ?? query[this.table][0].account_id; + this.project_id = project_id; } - this.fields = keys(query[table][0]).filter( - (field) => !this.primaryKeysSet.has(field), - ); - this.readData(); + this.primaryKeys = client_db.primary_keys(this.table); } - init = async () => { - await this.readData(); - }; - - get = (obj?) => { - if (this.state != "connected") { - throw Error("must be connected"); - } - if (obj == null) { - const result: any = {}; - for (const k in this.data) { - result[this.primaryString(this.data[k])] = this.data[k]; - } - return this.getHook(result); - } - return this.getHook(this.data[this.getKey(obj)]); - }; - - get_one = () => { - for (const key in this.data) { - return this.getHook(this.data[key]); - } - }; - - set = (obj) => { - if (Map.isMap(obj)) { - obj = obj.toJS(); - } - obj = this.fillInFromQuery(obj); - const key = this.getKey(obj); - this.data[key] = { ...this.data[key], ...obj }; - this.setToKv(obj); - }; - - save = async () => { - // TODO -- right now it is instantly saving on any change... - }; - - delete = (obj) => { - if (Map.isMap(obj)) { - obj = obj.toJS(); - } - const key = this.getKey(obj); - delete this.data[key]; - this.deleteFromKv(obj); - }; - - close = () => { - this.state = "closed"; - this.emit(this.state); - this.removeAllListeners(); - this.updateListener?.stop(); - this.data = {}; + private set_state = (state: State): void => { + this.state = state; + this.emit(state); }; get_state = () => { return this.state; }; - public async wait(until: Function, timeout: number = 30): Promise { - if (this.state == "closed") { - throw Error("wait: must not be closed"); + private getName = () => { + const primary: any = {}; + const spec = this.query[this.table][0]; + for (const key of this.primaryKeys) { + const val = spec[key]; + if (val != null) { + primary[key] = val; + } } - return await wait({ - obj: this, - until, - timeout, - change_event: "change-no-throttle", - }); - } + if (Object.keys(primary).length == 0) { + return this.table; + } else { + return `${this.table}-${jsonStableStringify(primary)}`; + } + }; - private getKv = reuseInFlight(async () => { - if (this.kv == null) { - this.kv = await getKv({ - nc: this.nc, + init = async () => { + const name = this.getName(); + if (this.atomic) { + this.dkv = await createDkv({ + name, + account_id: this.account_id, project_id: this.project_id, + env: this.env, + }); + } else { + this.dkv = await createDko({ + name, account_id: this.account_id, + project_id: this.project_id, + env: this.env, }); } - return this.kv!; - }); - - // load initial data - private readData = reuseInFlight(async () => { - this.data = await this.getFromKv(); - this.state = "connected"; - this.emit(this.state); - this.listenForUpdates(); - }); - - private listenForUpdates = async () => { - const kv = await this.getKv(); - this.updateListener = await kv.watch({ - key: `${this.natsKeyPrefix}.>`, - }); - for await (const { key, value, update } of this.updateListener) { - const i = key.lastIndexOf("."); - const field = key.slice(i + 1); - const prefix = key.slice(0, i); - if (this.data[prefix] == null && value.length > 0) { - this.data[prefix] = {}; - } - const s = this.data[prefix]; - if (update && s != null && Object.keys(s).length > 0) { - let k; - try { - k = this.primaryString(s); - } catch { - // s could be {} or just not enough if filled in to compute key. - k = null; - } - if (k != null) { - this.emit("change-no-throttle", [k]); - this.changedKeys.add(k); - this.throttledChangeEvent(); - } - } - if (s != null) { - if (value.length == 0 && this.primaryKeysSet.has(field)) { - delete this.data[prefix]; - } else { - if (this.allFields.has(field)) { - s[field] = this.jc.decode(value); - } - if (Object.keys(s).length == 0) { - delete this.data[prefix]; - } - } + this.dkv.on("change", (x) => { + if (!this.atomic) { + x = { ...x, value: this.dkv?.get(x.key) }; } - } + this.emit("change", x); + }); + this.set_state("connected"); }; - // this is throttled in constructor - private throttledChangeEvent = () => { - if (this.changedKeys.size > 0) { - this.emit("change", Array.from(this.changedKeys)); - this.changedKeys.clear(); + getKey = (obj_or_key): string => { + if (typeof obj_or_key == "string") { + return obj_or_key; + } + let obj = obj_or_key; + if (Map.isMap(obj)) { + obj = obj.toJS(); } - }; - - private fillInFromQuery = (obj) => { - return { ...obj, ...this.specifiedByQuery }; - }; - - private primaryString = (obj): string => { if (this.primaryKeys.length === 1) { - const k = obj[this.primaryKeys[0]]; - if (k == null) { - throw Error(`primary key '${this.primaryKeys[0]}' not set for object`); - } - return toKey(k)!; + return toKey(obj[this.primaryKeys[0]] ?? "")!; } else { // compound primary key - return toKey( - this.primaryKeys.map((pk) => { - const v = obj[pk]; - if (v == null) { - console.log({ obj }); - throw Error( - `part of compound primary key '${pk}' not set for object`, - ); - } - return v; - }), - )!; + return toKey(this.primaryKeys.map((pk) => obj[pk]))!; } }; - private natObjectKey = (obj): string => { - if (obj == null) { - throw Error("obj must be an object (not null)"); + set = (obj) => { + if (this.dkv == null) throw Error("closed"); + if (Map.isMap(obj)) { + obj = obj.toJS(); } - return this.sha1(this.primaryString(this.fillInFromQuery(obj))); + this.dkv.set(this.getKey(obj), obj); }; - getKey = (obj, field?: string): string => { - const x = this.singleton - ? this.natsKeyPrefix - : `${this.natsKeyPrefix}.${this.natObjectKey(obj)}`; - if (field == null) { - return x; - } else { - return `${x}.${field}`; - } + delete = (obj_or_key) => { + if (this.dkv == null) throw Error("closed"); + this.dkv.delete(this.getKey(obj_or_key)); }; - private setToKv = async (obj) => { - const kv = await this.getKv(); - const key = this.getKey(obj); - for (const field in obj) { - const value = this.jc.encode(obj[field]); - await kv.put(`${key}.${field}`, value); + get = (obj_or_key?) => { + if (this.dkv == null) throw Error("closed"); + if (obj_or_key == null) { + return this.getHook(this.dkv.get()); } + return this.getHook(this.dkv.get(this.getKey(obj_or_key))); }; - private deleteFromKv = async (obj) => { - const kv = await this.getKv(); - const key = this.getKey(obj); - const keys = await kv.keys(`${key}.>`); - for await (const k of keys) { - await kv.delete(k); + get_one = () => { + if (this.dkv == null) throw Error("closed"); + // TODO: insanely inefficient, especially if !atomic! + for (const key in this.dkv.get()) { + return this.get(key); } - await kv.delete(key); }; - getFromKv = async (obj?, field?) => { - const kv = await this.getKv(); - if (obj == null) { - // everything known in this table by the project - const { all: raw } = await getAllFromKv({ - kv, - key: `${this.natsKeyPrefix}.>`, - }); - const all: any = {}; - for (const key in raw) { - const x = raw[key]; - if (x) { - const val = this.jc.decode(x); - const i = key.lastIndexOf("."); - const field = key.slice(i + 1); - if (!this.allFields.has(field)) { - continue; - } - const prefix = key.slice(0, i); - if (all[prefix] == null) { - all[prefix] = {}; - } - const s = all[prefix]; - s[field] = val; - } - } - return all; - } - if (field == null) { - const s = { ...obj }; - const key = this.getKey(obj); - let nontrivial = false; - // todo: possibly better to just ask for everything under ${key}.> - // and take what is needed? Not sure. - for (const field of this.fields) { - const mesg = await kv.get(`${key}.${field}`); - const val = mesg?.sm?.data ? this.jc.decode(mesg.sm.data) : null; - if (val != null) { - s[field] = val; - nontrivial = true; - } - } - return nontrivial ? s : undefined; - } - const mesg = await kv.get(this.getKey(obj, field)); - if (mesg == null) { - return undefined; - } - return this.jc.decode(mesg.sm.data); + save = async () => { + await this.dkv?.save(); }; - // watch for changes in ONE object - async *watchOne(obj) { - const kv = await this.getKv(); - const w = await kv.watch({ - key: this.getKey(this.getKey(obj), "*"), - }); - for await (const { key, value } of w) { - const field = key.slice(key.lastIndexOf(".") + 1); - yield { [field]: this.jc.decode(value) }; + close = () => { + if (this.state == "closed") return; + this.set_state("closed"); + this.removeAllListeners(); + this.dkv?.close(); + delete this.dkv; + // @ts-ignore + delete this.env; + }; + + public async wait(until: Function, timeout: number = 30): Promise { + if (this.state == "closed") { + throw Error("wait: must not be closed"); } + return await wait({ + obj: this, + until, + timeout, + change_event: "change", + }); } } diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts index 498ceb8d93..35cdca01ae 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/nats/sync/synctable-stream.ts @@ -14,6 +14,7 @@ import { client_db } from "@cocalc/util/db-schema/client-db"; import { EventEmitter } from "events"; import { type NatsEnv } from "@cocalc/nats/types"; import { dstream, DStream } from "./dstream"; +import { fromJS, Map } from "immutable"; export type State = "disconnected" | "connected" | "closed"; @@ -37,19 +38,23 @@ export class SyncTableStream extends EventEmitter { private state: State = "disconnected"; private env; private dstream?: DStream; + private getHook: Function; constructor({ query, env, account_id: _account_id, project_id, + immutable, }: { query; env: NatsEnv; account_id?: string; project_id?: string; + immutable?: boolean; }) { super(); + this.getHook = immutable ? fromJS : (x) => x; this.env = env; const table = keys(query)[0]; this.table = table; @@ -106,6 +111,9 @@ export class SyncTableStream extends EventEmitter { getKey = this.primaryString; set = (obj) => { + if (Map.isMap(obj)) { + obj = obj.toJS(); + } // console.log("set", obj); // delete string_id since it is redundant info const key = this.primaryString(obj); @@ -136,18 +144,17 @@ export class SyncTableStream extends EventEmitter { get = (obj?) => { if (obj == null) { - // CAREFUL - return this.data; + return this.getHook(this.data); } if (typeof obj == "string") { - return this.data[obj]; + return this.getHook(this.data[obj]); } if (is_array(obj)) { const x: any = {}; for (const key of obj) { x[this.primaryString(key)] = this.get(key); } - return x; + return this.getHook(x); } let key; if (typeof obj == "object") { @@ -155,7 +162,7 @@ export class SyncTableStream extends EventEmitter { } else { key = `${key}`; } - return this.data[key]; + return this.getHook(this.data[key]); }; getSortedTimes = () => { diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/nats/sync/synctable.ts index 20541d7cc4..bef9d83eba 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/nats/sync/synctable.ts @@ -1,9 +1,8 @@ import { type NatsEnv } from "@cocalc/nats/types"; import { SyncTableKV } from "./synctable-kv"; -import { SyncTableKVAtomic } from "./synctable-kv-atomic"; import { SyncTableStream } from "./synctable-stream"; -export type SyncTable = SyncTableKV | SyncTableStream | SyncTableKVAtomic; +export type SyncTable = SyncTableStream | SyncTableKV; // When the database is watching tables for changefeeds, if it doesn't get a clear expression // of interest from a client every this much time, it automatically stops. @@ -28,21 +27,16 @@ export function createSyncTable({ immutable?: boolean; // if true, then get/set works with immutable.js objects instead. }) { if (stream) { - if (atomic === false) { - throw Error("streams must be atomic"); - } - if (immutable) { - throw Error("immutable not yet supported for streams"); - } return new SyncTableStream({ query, env, account_id, project_id, + immutable, ...options, }); } else { - return new SyncTableKVAtomic({ + return new SyncTableKV({ query, env, account_id, diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index 4f5636bf23..10aa4946c1 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -95,11 +95,12 @@ async function listen(api, subject) { async function handleApiRequest(request, mesg) { let resp; + const { name, args } = request as any; try { - const { name, args } = request as any; - logger.debug("handling project.api request:", { name }); + // logger.debug("handling project.api request:", { name }); resp = (await getResponse({ name, args })) ?? null; } catch (err) { + logger.debug(`project.api request err = ${err}`, { name }); resp = { error: `${err}` }; } mesg.respond(jc.encode(resp)); diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index e7c7c6a1f5..61bbf7d6b1 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -46,11 +46,24 @@ export class NatsChangefeed extends EventEmitter { }; private interest = async () => { - await delay(30000); + let d = 10000; + await delay(d); while (this.state != "closed") { // console.log("express interest in", this.query); - await this.client.nats_client.changefeedInterest(this.query); - await delay(30000); + try { + await this.client.nats_client.changefeedInterest(this.query); + d = Math.min(45000, 1.3 * d) + Math.random(); + } catch (err) { + if (err.code != "TIMEOUT") { + // it's normal for this to throw a TIMEOUT error whenever the browser isn't connected to NATS, + // so we only log it to the console if it is unexpected. + console.log("WARNING: issue updating changefeed interest", err); + } else { + // reset to be more frequently since likely disconnected. + d = 10000; + } + } + await delay(d); } }; @@ -59,7 +72,12 @@ export class NatsChangefeed extends EventEmitter { return; } this.natsSynctable.on("change", ({ value: new_val, prev: old_val }) => { - this.emit("update", { action: "update", new_val, old_val }); + // console.log("natsSynctable, change, ", { new_val, old_val }); + if (new_val == null) { + this.emit("delete", { action: "delete", old_val }); + } else { + this.emit("update", { action: "update", new_val, old_val }); + } }); }; } From ac0b8ff2f36765db4e6ca367962a12698957ad1c Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 11 Feb 2025 00:43:01 +0000 Subject: [PATCH 165/281] nats: subtle issues with deleting from non-atomic distributed changefeeds and synctables - NOT fully working yet. --- src/packages/nats/sync/dko.ts | 12 +++++++++- src/packages/nats/sync/dkv.ts | 12 ++++++---- src/packages/nats/sync/general-dkv.ts | 2 +- src/packages/nats/sync/general-kv.ts | 9 +------ src/packages/nats/sync/kv.ts | 4 +++- src/packages/nats/sync/synctable-kv.ts | 8 ++++++- src/packages/sync/table/changefeed-nats.ts | 28 +++++++++++++++------- src/packages/sync/table/synctable.ts | 20 +++++++++------- 8 files changed, 62 insertions(+), 33 deletions(-) diff --git a/src/packages/nats/sync/dko.ts b/src/packages/nats/sync/dko.ts index d8ee62e2b0..d8c30ae835 100644 --- a/src/packages/nats/sync/dko.ts +++ b/src/packages/nats/sync/dko.ts @@ -66,16 +66,25 @@ export class DKO extends EventEmitter { }); this.dkv.on("change", ({ key: path, value }) => { if (path == null) { - // TODO: why would this happen? + // TODO: could this happen? return; } const { key, field } = this.fromPath(path); if (!field) { + // there is no field part of the path, which happens + // only for delete of entire object, after setting all + // the fields to null. this.emit("change", { key }); } else { + if (value === undefined && this.dkv?.get(key) == null) { + // don't emit change setting fields to undefined if the + // object was already deleted. + return; + } this.emit("change", { key, field, value }); } }); + this.dkv.on("reject", ({ key: path, value }) => { if (path == null) { // TODO: would this happen? @@ -118,6 +127,7 @@ export class DKO extends EventEmitter { if (fields == null) { return; } + this.dkv.delete(key); for (const field of fields) { this.dkv.delete(this.toPath(key, field)); } diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index ed2adacbc0..bf414e0646 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -13,7 +13,7 @@ Type ".help" for more information. import { EventEmitter } from "events"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { GeneralDKV, type MergeFunction } from "./general-dkv"; +import { GeneralDKV, TOMBSTONE, type MergeFunction } from "./general-dkv"; import { userKvKey, type KVOptions } from "./kv"; import { jsName } from "@cocalc/nats/names"; import { sha1 } from "@cocalc/util/misc"; @@ -84,11 +84,15 @@ export class DKV extends EventEmitter { } this.generalDKV = new GeneralDKV(this.opts); this.generalDKV.on("change", ({ value, prev }) => { - if (value != null) { - this.emit("change", { key: value.key, value: value.value }); + if (value != null && value !== TOMBSTONE) { + this.emit("change", { + key: value.key, + value: value.value, + prev: prev?.value, + }); } else if (prev != null) { // value is null so it's a delete - this.emit("change", { key: prev.key }); + this.emit("change", { key: prev.key, prev: prev.value }); } }); this.generalDKV.on("reject", ({ value }) => { diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index 6cc18b713a..f4ea36130d 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -78,7 +78,7 @@ import { isEqual } from "lodash"; import { delay } from "awaiting"; import { map as awaitMap } from "awaiting"; -const TOMBSTONE = Symbol("tombstone"); +export const TOMBSTONE = Symbol("tombstone"); const MAX_PARALLEL = 50; export type MergeFunction = (opts: { diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index d5327c7a13..336bc9af93 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -301,19 +301,12 @@ export class GeneralKV extends EventEmitter { if (this.all[key] !== undefined) { const cur = this.all[key]; try { - delete this.all[key]; const newRevision = await this.kv.delete(key, { previousSeq: revision ?? this.revisions[key], }); this.revisions[key] = newRevision; - delete this.times[key]; - delete this.sizes[key]; } catch (err) { - if (cur === undefined) { - delete this.all[key]; - } else { - this.all[key] = cur; - } + this.all[key] = cur; throw err; } } diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index 042955e7fe..24362d9830 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -1,10 +1,12 @@ /* Always Consistent Centralized Key Value Store +NOTE: I think this isn't used by anything actually. Note it doesn't emit +change events. Maybe we should delete this. DEVELOPMENT: -~/cocalc/src/packages/backend n +~/cocalc/src/packages/backend$ n Welcome to Node.js v18.17.1. Type ".help" for more information. > t = await require("@cocalc/backend/nats/sync").kv({name:'test'}) diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index bf22305fc2..b023b1dc76 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -102,7 +102,13 @@ export class SyncTableKV extends EventEmitter { } this.dkv.on("change", (x) => { if (!this.atomic) { - x = { ...x, value: this.dkv?.get(x.key) }; + if (x.value === undefined) { + // delete + x = { ...x, prev: this.dkv?.get(x.key) }; + } else { + // change + x = { ...x, value: this.dkv?.get(x.key) }; + } } this.emit("change", x); }); diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index 61bbf7d6b1..5b67e31091 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -6,6 +6,8 @@ import { EventEmitter } from "events"; import type { State } from "./changefeed"; import { delay } from "awaiting"; +//import { CHANGEFEED_INTEREST_PERIOD_MS } from "@cocalc/nats/sync/synctable"; +const CHANGEFEED_INTEREST_PERIOD_MS = 120000; export class NatsChangefeed extends EventEmitter { private client; @@ -52,7 +54,8 @@ export class NatsChangefeed extends EventEmitter { // console.log("express interest in", this.query); try { await this.client.nats_client.changefeedInterest(this.query); - d = Math.min(45000, 1.3 * d) + Math.random(); + d = + Math.min(CHANGEFEED_INTEREST_PERIOD_MS / 3, 1.3 * d) + Math.random(); } catch (err) { if (err.code != "TIMEOUT") { // it's normal for this to throw a TIMEOUT error whenever the browser isn't connected to NATS, @@ -71,13 +74,20 @@ export class NatsChangefeed extends EventEmitter { if (this.natsSynctable == null) { return; } - this.natsSynctable.on("change", ({ value: new_val, prev: old_val }) => { - // console.log("natsSynctable, change, ", { new_val, old_val }); - if (new_val == null) { - this.emit("delete", { action: "delete", old_val }); - } else { - this.emit("update", { action: "update", new_val, old_val }); - } - }); + this.natsSynctable.on( + "change", + ({ key, value: new_val, prev: old_val }) => { + let x; + if (new_val == null) { + x = { action: "delete", old_val, key }; + } else if (old_val !== undefined) { + x = { action: "update", new_val, old_val, key }; + } else { + x = { action: "insert", new_val, key }; + } + console.log("natsSynctable, change, ", x); + this.emit("update", x); + }, + ); }; } diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index 4cee800248..067dcebb50 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -1563,6 +1563,7 @@ export class SyncTable extends EventEmitter { change.old_val, change.action, this.coerce_types, + change.key, ); if (key != null) { changed_keys.push(key); @@ -1594,6 +1595,7 @@ export class SyncTable extends EventEmitter { old_val: any, action: string, coerce: boolean, + key?: string, ): string | undefined { if (this.value == null) { // to satisfy typescript. @@ -1601,14 +1603,16 @@ export class SyncTable extends EventEmitter { } if (action === "delete") { - old_val = fromJS(old_val); - if (old_val == null) { - throw Error("old_val must not be null for delete action"); - } - if (coerce && this.coerce_types) { - old_val = this.do_coerce_types(old_val); + if (!key) { + old_val = fromJS(old_val); + if (old_val == null) { + throw Error("old_val must not be null for delete action"); + } + if (coerce && this.coerce_types) { + old_val = this.do_coerce_types(old_val); + } + key = this.obj_to_key(old_val); } - const key = this.obj_to_key(old_val); if (key == null || !this.value.has(key)) { return; // already gone } @@ -1623,7 +1627,7 @@ export class SyncTable extends EventEmitter { if (coerce && this.coerce_types) { new_val = this.do_coerce_types(new_val); } - const key = this.obj_to_key(new_val); + key = this.obj_to_key(new_val); if (key == null) { // This means the primary key is null or missing, which // shouldn't happen. Maybe it could in some edge case. From 5e30be6636c5c9c952cfc812b336c5029e3151af Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 11 Feb 2025 02:29:31 +0000 Subject: [PATCH 166/281] messages table: don't merge in messages, since then deleting doesn't work --- src/packages/frontend/messages/redux.ts | 8 +------- src/packages/sync/table/synctable.ts | 4 +++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/packages/frontend/messages/redux.ts b/src/packages/frontend/messages/redux.ts index 719ed7e8da..92a0a53269 100644 --- a/src/packages/frontend/messages/redux.ts +++ b/src/packages/frontend/messages/redux.ts @@ -137,14 +137,8 @@ export class MessagesActions extends Actions { } }; - handleTableUpdate = (updatedMessages) => { + handleTableUpdate = (messages) => { const store = this.getStore(); - let messages = store.get("messages"); - if (messages == null) { - messages = updatedMessages; - } else { - messages = messages.merge(updatedMessages); - } messages = getNotExpired(messages); const threads = getThreads(messages); this.setState({ messages, threads }); diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index 067dcebb50..56c0763de2 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -1606,7 +1606,9 @@ export class SyncTable extends EventEmitter { if (!key) { old_val = fromJS(old_val); if (old_val == null) { - throw Error("old_val must not be null for delete action"); + throw Error( + "old_val must not be null or key must be specified for delete action", + ); } if (coerce && this.coerce_types) { old_val = this.do_coerce_types(old_val); From d8f25dceff9d49fbab79763c6a83bcd3b49e99a9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 11 Feb 2025 04:07:02 +0000 Subject: [PATCH 167/281] nats: greatly speedup reading initial kv out of nats --- src/packages/database/nats/changefeeds.ts | 7 ++- src/packages/nats/util.ts | 71 ++++++++-------------- src/packages/sync/table/changefeed-nats.ts | 5 +- 3 files changed, 34 insertions(+), 49 deletions(-) diff --git a/src/packages/database/nats/changefeeds.ts b/src/packages/database/nats/changefeeds.ts index fbbe2f98f1..7084ffbfcb 100644 --- a/src/packages/database/nats/changefeeds.ts +++ b/src/packages/database/nats/changefeeds.ts @@ -153,16 +153,21 @@ const createChangefeed = reuseInFlight( const changes = uuid(); changefeedHashes[changes] = hash; const env = { nc, jc, sha1 }; + // If you change any settings below, you might also have to change them in + // src/packages/sync/table/changefeed-nats.ts const synctable = createSyncTable({ query, env, account_id: opts.account_id, project_id: opts.project_id, - atomic: false, + // atomic = false is just way too slow due to the huge number of distinct + // messages, which NATS is not as good with. + atomic: true, immutable: false, }); await synctable.init(); + // if (global.z == null) { // global.z = {}; // } diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index 92781448f3..50b07e3377 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -1,70 +1,49 @@ import jsonStableStringify from "json-stable-stringify"; // Get the number of NON-deleted keys in a nats kv store, matching a given subject: -export async function numKeys(kv, x: string | string[] = ">"): Promise { - let num = 0; - for await (const _ of await kv.keys(x)) { - num += 1; - } - return num; -} +// export async function numKeys(kv, x: string | string[] = ">"): Promise { +// let num = 0; +// for await (const _ of await kv.keys(x)) { +// num += 1; +// } +// return num; +// } // get everything from a KV store matching a subject pattern. -// TRICK! Note that getting the keys, then the value -// for each key, which is what the JS API docs suggests (?)... -// is **INSANELY SLOW**! -// Instead count the keys, then use watch and stop when we have them all. -// It's ridiculous but fast, and is *slightly* dangerous, since the size -// of the kv store changed maybe right after computing the size, but -// it's a risk we must take. We put in a 5s default timeout to avoid -// any possibility of hanging forever as a result. export async function getAllFromKv({ kv, key = ">", - timeout = 5000, }: { kv; key?: string | string[]; - timeout?: number; }): Promise<{ all: { [key: string]: any }; revisions: { [key: string]: number }; times: { [key: string]: Date }; }> { - const total = await numKeys(kv, key); - let count = 0; + // const t = Date.now(); + // console.log("start getAllFromKv", key); const all: any = {}; const revisions: { [key: string]: number } = {}; const times: { [key: string]: Date } = {}; - if (total == 0) { - return { all, revisions, times }; - } - const watch = await kv.watch({ key, ignoreDeletes: true }); - let id: any = 0; - for await (const { key, value, revision, sm } of watch) { - all[key] = value; - revisions[key] = revision; - times[key] = sm.time; - - count += 1; - - if (id) { - clearTimeout(id); - id = 0; - } - if (count >= total) { - break; + const watch = await kv.watch({ key, ignoreDeletes: false }); + if (watch._data._info.num_pending > 0) { + for await (const { key, value, revision, sm } of watch) { + if (value.length > 0) { + // we MUST check value.length because we do NOT ignoreDeletes. + // we do NOT ignore deletes so that sm.di.pending goes down to 0. + // Otherwise, there is no way in general to know when we are done. + all[key] = value; + revisions[key] = revision; + times[key] = sm.time; + } + // console.log("getAllFromKv", key, sm.di.pending); + if (sm.di.pending <= 0) { + break; + } } - // make a timeout so if the wait from one iteration to the - // next in the loop is more than this amount, it stops. - // This should never happen unless the network were somehow VERY slow - // or the kv size shrunk at the exactly wrong time (and even then, - // it might work due to delete notifications). This is only about - // getting data from NATS, not the database, so should always be fast. - id = setTimeout(() => { - watch.stop(); - }, timeout); } + // console.log("finished getAllFromKv", key, (Date.now() - t) / 1000, "seconds"); return { all, revisions, times }; } diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index 5b67e31091..b2c7d09e49 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -28,7 +28,9 @@ export class NatsChangefeed extends EventEmitter { connect = async () => { this.natsSynctable = await this.client.nats_client.changefeed(this.query, { - atomic: false, + // atomic=false means less data transfer on changes, but simply does not scale up + // well and is hence quite slow overall. + atomic: true, immutable: false, }); this.interest(); @@ -85,7 +87,6 @@ export class NatsChangefeed extends EventEmitter { } else { x = { action: "insert", new_val, key }; } - console.log("natsSynctable, change, ", x); this.emit("update", x); }, ); From 6115ccc89a9a3ea1945fda2079b41f5ae16d77cd Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 11 Feb 2025 14:15:08 +0000 Subject: [PATCH 168/281] nats install: improve install process, document better, newest nats-server --- src/packages/backend/nats/install.ts | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/packages/backend/nats/install.ts b/src/packages/backend/nats/install.ts index 2b7dfed270..e799096b34 100644 --- a/src/packages/backend/nats/install.ts +++ b/src/packages/backend/nats/install.ts @@ -8,6 +8,21 @@ for this architecture: - nsc We assume curl and python3 are installed. + +DEVELOPMENT: + +Installation happens automatically, e.g,. when you do 'pnpm nats-server' or +start the hub via 'pnpm hub'. However, you can explicitly do +an install as follows: + +~/cocalc/src/packages/backend/nats$ DEBUG=cocalc:* DEBUG_CONSOLE=yes node +Welcome to Node.js v18.17.1. +Type ".help" for more information. +> await require('@cocalc/backend/nats/install').install() + +Installing just the server: + +> await require('@cocalc/backend/nats/install').installNatsServer() */ import { nats } from "@cocalc/backend/data"; @@ -17,7 +32,9 @@ import { executeCode } from "@cocalc/backend/execute-code"; import getLogger from "@cocalc/backend/logger"; const VERSIONS = { - "nats-server": "v2.11.0-preview.2", + // https://github.com/nats-io/nats-server/releases + "nats-server": "v2.10.26-RC.2", + // https://github.com/nats-io/natscli/releases nats: "v0.1.6", }; @@ -62,7 +79,7 @@ async function getVersion(name: string) { } } -async function installNatsServer(noUpgrade) { +export async function installNatsServer(noUpgrade) { if (noUpgrade && (await pathExists(join(bin, "nats-server")))) { return; } @@ -72,9 +89,10 @@ async function installNatsServer(noUpgrade) { ); return; } - logger.debug("installing nats-server"); + const command = `curl -sf https://binaries.nats.dev/nats-io/nats-server/v2@${VERSIONS["nats-server"]} | sh`; + logger.debug("installing nats-server: ", command); await executeCode({ - command: `curl -sf https://binaries.nats.dev/nats-io/nats-server/v2@${VERSIONS["nats-server"]} | sh`, + command, path: bin, verbose: true, }); @@ -113,7 +131,7 @@ export async function installNsc(noUpgrade) { } } -async function installNatsCli(noUpgrade) { +export async function installNatsCli(noUpgrade) { if (noUpgrade && (await pathExists(join(bin, "nats")))) { return; } From 13e61f0196f2395bce9ef06295d1822d5ce60025 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 11 Feb 2025 14:39:40 +0000 Subject: [PATCH 169/281] nats: start unit testing; even found a bug in creating a change handler. --- src/packages/backend/nats/sync.ts | 8 +-- src/packages/backend/nats/test/dkv.test.ts | 61 ++++++++++++++++++++++ src/packages/backend/package.json | 2 +- src/packages/nats/sync/dko.ts | 2 +- src/packages/nats/sync/dkv.ts | 18 +++++-- src/packages/nats/sync/dstream.ts | 21 ++++++-- src/packages/nats/sync/kv.ts | 2 +- 7 files changed, 98 insertions(+), 16 deletions(-) create mode 100644 src/packages/backend/nats/test/dkv.test.ts diff --git a/src/packages/backend/nats/sync.ts b/src/packages/backend/nats/sync.ts index 5c346aa78b..41bf4485de 100644 --- a/src/packages/backend/nats/sync.ts +++ b/src/packages/backend/nats/sync.ts @@ -14,16 +14,16 @@ export async function stream(opts): Promise { return await createStream({ env: await getEnv(), ...opts }); } -export async function dstream(opts): Promise { - return await createDstream({ env: await getEnv(), ...opts }); +export async function dstream(opts, options?): Promise { + return await createDstream({ env: await getEnv(), ...opts }, options); } export async function kv(opts): Promise { return await createKV({ env: await getEnv(), ...opts }); } -export async function dkv(opts): Promise { - return await createDKV({ env: await getEnv(), ...opts }); +export async function dkv(opts, options?): Promise { + return await createDKV({ env: await getEnv(), ...opts }, options); } export async function dko(opts): Promise { diff --git a/src/packages/backend/nats/test/dkv.test.ts b/src/packages/backend/nats/test/dkv.test.ts new file mode 100644 index 0000000000..19ad7c06bc --- /dev/null +++ b/src/packages/backend/nats/test/dkv.test.ts @@ -0,0 +1,61 @@ +import { dkv as createDkv } from "@cocalc/backend/nats/sync"; +import { once } from "@cocalc/util/async-utils"; + +describe("create a public dkv and do basic operations", () => { + let kv; + const name = `test-${Math.random()}`; + + it("creates the dkv", async () => { + kv = await createDkv({ name }); + expect(kv.get()).toEqual({}); + }); + + it("adds a key to the dkv", () => { + kv.a = 10; + expect(kv.a).toEqual(10); + }); + + it("waits for the dkv to be longterm saved, then closing and recreates the kv and verifies that the key is there.", async () => { + await kv.save(); + kv.close(); + kv = await createDkv({ name }); + expect(kv.a).toEqual(10); + }); + + it("closes the kv", async () => { + kv.close(); + expect(kv.get).toThrow("closed"); + }); +}); + +describe("opens a dkv twice at once (disabling caching) and observe sync", () => { + let kv1; + let kv2; + const name = `test-${Math.random()}`; + + it("creates the dkv twice", async () => { + kv1 = await createDkv({ name }, { noCache: true }); + kv2 = await createDkv({ name }, { noCache: true }); + expect(kv1.get()).toEqual({}); + expect(kv2.get()).toEqual({}); + expect(kv1 === kv2).toBe(false); + }); + + it("sets a value in one and sees that it is NOT instantly set in the other", () => { + kv1.a = 25; + expect(kv2.a).toBe(undefined); + }); + + it("awaits save and then sees the value *eventually* appears in the other", async () => { + await kv1.save(); + // initially not there. + expect(kv2.a).toBe(undefined); + await once(kv2, "change"); + expect(kv2.a).toBe(kv1.a); + }); + + it("close up", () => { + kv1.close(); + kv2.close(); + }); +}); diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index ddb8a4dde9..991f41cdb5 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -16,7 +16,7 @@ "clean": "rm -rf dist node_modules", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", - "test": "pnpm exec jest --detectOpenHandles", + "test": "pnpm exec jest --forceExit --detectOpenHandles", "prepublishOnly": "pnpm test" }, "files": ["dist/**", "bin/**", "README.md", "package.json"], diff --git a/src/packages/nats/sync/dko.ts b/src/packages/nats/sync/dko.ts index d8c30ae835..15daead6e8 100644 --- a/src/packages/nats/sync/dko.ts +++ b/src/packages/nats/sync/dko.ts @@ -40,7 +40,7 @@ export class DKO extends EventEmitter { }, set(target, prop, value) { prop = String(prop); - if (prop == "_eventsCount") { + if (prop == "_eventsCount" || prop == "_events") { target[prop] = value; return true; } diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index bf414e0646..71a20a49a2 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -62,7 +62,7 @@ export class DKV extends EventEmitter { }, set(target, prop, value) { prop = String(prop); - if (prop == "_eventsCount") { + if (prop == "_eventsCount" || prop == "_events") { target[prop] = value; return true; } @@ -188,17 +188,25 @@ export class DKV extends EventEmitter { const cache: { [key: string]: DKV } = {}; export const dkv = reuseInFlight( - async (opts: DKVOptions) => { - const key = userKvKey(opts); - if (cache[key] == null) { + async (opts: DKVOptions, { noCache }: { noCache?: boolean } = {}) => { + const f = async () => { const k = new DKV(opts); await k.init(); + return k; + }; + if (noCache) { + // especially useful for unit testing. + return await f(); + } + const key = userKvKey(opts); + if (cache[key] == null) { + const k = await f(); k.on("closed", () => delete cache[key]); cache[key] = k; } return cache[key]!; }, { - createKey: (args) => userKvKey(args[0]), + createKey: (args) => userKvKey(args[0]) + JSON.stringify(args[1]), }, ); diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 5a4002bb52..85ffb94cab 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -193,13 +193,15 @@ export class DStream extends EventEmitter { const dstreamCache: { [key: string]: DStream } = {}; export const dstream = reuseInFlight( - async (options: UserStreamOptions) => { + async ( + options: UserStreamOptions, + { noCache }: { noCache?: boolean } = {}, + ) => { const { account_id, project_id, name } = options; const jsname = jsName({ account_id, project_id }); const subjects = streamSubject({ account_id, project_id }); const filter = subjects.replace(">", (options.env.sha1 ?? sha1)(name)); - const key = userStreamOptionsKey(options); - if (dstreamCache[key] == null) { + const f = async () => { const dstream = new DStream({ ...options, name, @@ -209,6 +211,16 @@ export const dstream = reuseInFlight( filter, }); await dstream.init(); + return dstream; + }; + if (noCache) { + // especially useful for unit testing. + return await f(); + } + + const key = userStreamOptionsKey(options); + if (dstreamCache[key] == null) { + const dstream = await f(); dstreamCache[key] = dstream; dstream.on("closed", () => { delete dstreamCache[key]; @@ -217,6 +229,7 @@ export const dstream = reuseInFlight( return dstreamCache[key]; }, { - createKey: (args) => userStreamOptionsKey(args[0]), + createKey: (args) => + userStreamOptionsKey(args[0]) + JSON.stringify(args[1]), }, ); diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index 24362d9830..5f50cd1f54 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -55,7 +55,7 @@ export class KV extends EventEmitter { }, set(target, prop, value) { prop = String(prop); - if (prop == "_eventsCount") { + if (prop == "_eventsCount" || prop == "_events") { target[prop] = value; return true; } From 664afdf1df5e2fe837b3add40c62475ddb95ba04 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 11 Feb 2025 16:55:50 +0000 Subject: [PATCH 170/281] nats dkv testing -- work in progress unit testing - discovering and working on fixing bugs! --- src/packages/backend/nats/test/dkv.test.ts | 172 ++++++++++++++++++++- src/packages/nats/sync/dkv.ts | 15 ++ src/packages/nats/sync/general-dkv.ts | 57 ++++++- src/packages/nats/sync/general-kv.ts | 4 + 4 files changed, 241 insertions(+), 7 deletions(-) diff --git a/src/packages/backend/nats/test/dkv.test.ts b/src/packages/backend/nats/test/dkv.test.ts index 19ad7c06bc..ba04b50a0c 100644 --- a/src/packages/backend/nats/test/dkv.test.ts +++ b/src/packages/backend/nats/test/dkv.test.ts @@ -28,7 +28,24 @@ describe("create a public dkv and do basic operations", () => { }); }); -describe("opens a dkv twice at once (disabling caching) and observe sync", () => { +describe("opens a dkv twice and verifies it was cached", () => { + let kv1; + let kv2; + const name = `test-${Math.random()}`; + + it("creates the dkv twice", async () => { + kv1 = await createDkv({ name }); + kv2 = await createDkv({ name }); + expect(kv1.get()).toEqual({}); + expect(kv1 === kv2).toBe(true); + }); + it("closes", async () => { + kv1.close(); + expect(kv2.get).toThrow("closed"); + }); +}); + +describe("opens a dkv twice at once and observe sync", () => { let kv1; let kv2; const name = `test-${Math.random()}`; @@ -59,3 +76,156 @@ describe("opens a dkv twice at once (disabling caching) and observe sync", () => kv2.close(); }); }); + +describe("check server assigned times", () => { + let kv; + const name = `test-${Math.random()}`; + + it("create a kv", async () => { + kv = await createDkv({ name }); + expect(kv.get()).toEqual({}); + expect(kv.time()).toEqual({}); + }); + + it("set a key, then get the time and confirm it is reasonable", async () => { + kv.a = { b: 7 }; + // not serve assigned yet + expect(kv.time("a")).toEqual(undefined); + await kv.save(); + // still not server assigned + expect(kv.time("a")).toEqual(undefined); + await once(kv, "change"); + // now we must have it. + // sanity check: within a second + expect(kv.time("a").getTime()).toBeCloseTo(Date.now(), -3); + // all the times + expect(Object.keys(kv.time()).length).toBe(1); + }); + + it("setting again with a *different* value changes the time", async () => { + kv.a = { b: 8 }; + const t0 = kv.time("a"); + await once(kv, "change"); + expect(kv.time("a").getTime()).toBeCloseTo(Date.now(), -3); + expect(t0).not.toEqual(kv.time("a")); + }); + + it("close", () => { + kv.close(); + }); +}); + +describe("test deleting and clearing a dkv", () => { + let kv1; + let kv2; + const name = `test-${Math.random()}`; + + const reset = () => { + kv1.clear(); + kv2.clear(); + }; + + it("creates the dkv twice without caching so can make sure sync works", async () => { + kv1 = await createDkv({ name }, { noCache: true }); + kv2 = await createDkv({ name }, { noCache: true }); + expect(kv1.get()).toEqual({}); + expect(kv2.get()).toEqual({}); + expect(kv1 === kv2).toBe(false); + }); + + it("adds an entry, deletes it and confirms", async () => { + kv1.foo = "bar"; + expect(kv1.has("foo")).toBe(true); + expect(kv2.has("foo")).toBe(false); + await once(kv2, "change"); + expect(kv2.foo).toBe(kv1.foo); + expect(kv2.has("foo")).toBe(true); + delete kv1.foo; + await once(kv2, "change"); + expect(kv2.foo).toBe(undefined); + expect(kv2.has("foo")).toBe(false); + }); + + it("adds an entry, clears it and confirms", async () => { + reset(); + + kv1.foo = "bar"; + await once(kv2, "change"); + expect(kv2.foo).toBe(kv1.foo); + kv2.clear(); + expect(kv2.has("foo")).toBe(false); + await once(kv1, "change"); + expect(kv1.has("foo")).toBe(false); + }); + + it("adds an entry, syncs, adds another local entry (not sync'd), clears in sync and confirms NOT everything was cleared", async () => { + reset(); + kv1.foo = Math.random(); + await kv1.save(); + if (kv2.foo != kv1.foo) { + await once(kv2, "change"); + } + expect(kv2.foo).toBe(kv1.foo); + kv1.xxx = "yyy"; + expect(kv2.xxx).toBe(undefined); + // this ONLY clears foo, not xxx + kv2.clear(); + await once(kv1, "change"); + expect(kv1.has("xxx")).toBe(true); + }); + + it("adds an entry, syncs, adds another local entry (not sync'd), clears in first one, and confirms everything was cleared", async () => { + reset(); + + kv1.foo = Math.random(); + await kv1.save(); + if (kv2.foo != kv1.foo) { + await once(kv2, "change"); + } + kv1.xxx = "yyy"; + expect(kv2.xxx).toBe(undefined); + // this ONLY clears foo, not xxx + kv1.clear(); + expect(kv1.has("xxx")).toBe(false); + }); +}); + +describe("set several items, confirm exist, save, and confirm they are still there", () => { + const name = `test-${Math.random()}`; + const count = 10; + it(`adds ${count} entries`, async () => { + const kv = await createDkv({ name }); + expect(kv.get()).toEqual({}); + for (let i = 0; i < count; i++) { + kv[`${i}`] = i; + } + console.log(kv.get()); + expect(Object.keys(kv.get()).length).toEqual(count); + await kv.save(); + console.log(kv.get()); + expect(Object.keys(kv.get()).length).toEqual(count); + }); +}); + +// import { delay } from "awaiting"; + +// describe("do a large insert and clear stress test", () => { +// const name = `test-${Math.random()}`; +// const count = 10; +// it(`adds ${count} entries, saves, clears, and confirms empty`, async () => { +// const kv = await createDkv({ name }); +// expect(kv.get()).toEqual({}); +// for (let i = 0; i < count; i++) { +// kv[`${i}`] = i; +// } +// console.log(kv.get()); +// expect(Object.keys(kv.get()).length).toEqual(count); +// await kv.save(); +// console.log(kv.get()); +// expect(Object.keys(kv.get()).length).toEqual(count); +// kv.clear(); +// expect(kv.get()).toEqual({}); +// await kv.save(); +// expect(kv.get()).toEqual({}); +// }); +// }); diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index 71a20a49a2..e0b9848d95 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -9,6 +9,10 @@ Welcome to Node.js v18.17.1. Type ".help" for more information. > t = await require("@cocalc/backend/nats/sync").dkv({name:'test'}) + +UNIT TESTS: See backend/nats/test/ + +They aren't right here, because this module doesn't have info to connect to NATS. */ import { EventEmitter } from "events"; @@ -122,6 +126,10 @@ export class DKV extends EventEmitter { this.generalDKV.delete(`${this.prefix}.${this.sha1(key)}`); }; + clear = () => { + this.generalDKV?.clear(); + }; + // server assigned time time = (key?: string) => { if (this.generalDKV == null) { @@ -142,6 +150,13 @@ export class DKV extends EventEmitter { return x; }; + has = (key: string) => { + if (this.generalDKV == null) { + throw Error("closed"); + } + return this.generalDKV.has(`${this.prefix}.${this.sha1(key)}`); + }; + get = (key?) => { if (this.generalDKV == null) { throw Error("closed"); diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index f4ea36130d..6dc5c01953 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -91,7 +91,11 @@ export type MergeFunction = (opts: { export class GeneralDKV extends EventEmitter { private kv?: GeneralKV; private merge?: MergeFunction; + // local values that have NOT been saved to NATS: private local: { [key: string]: any } = {}; + // local values that HAVE been saved to NATS but may not yet be in this.kv.get() + private localSaved: { [key: string]: any } = {}; + // these may have been changed locally: private changed: Set = new Set(); constructor({ @@ -148,13 +152,15 @@ export class GeneralDKV extends EventEmitter { }; private handleRemoteChange = ({ key, value: remote, prev }) => { - const local = this.local[key]; + // local = value we have NOT saved: + const local = this.local[key] ?? this.localSaved[key]; let value: any = remote; if (local !== undefined) { if (isEqual(local, remote)) { // we have a local change, but it's the same change as remote, so just // forget about our local change. delete this.local[key]; + delete this.localSaved[key]; } else { try { value = this.merge?.({ key, local, remote, prev }) ?? local; @@ -173,9 +179,11 @@ export class GeneralDKV extends EventEmitter { if (isEqual(value, remote)) { // no change, so forget our local value delete this.local[key]; + delete this.localSaved[key]; } else { // resolve with the new value, or if it is undefined, a TOMBSTONE, meaning choice is to delete. this.local[key] = value ?? TOMBSTONE; + delete this.localSaved[key]; } } } @@ -188,7 +196,7 @@ export class GeneralDKV extends EventEmitter { } if (key != null) { this.assertValidKey(key); - const local = this.local[key]; + const local = this.local[key] ?? this.localSaved[key]; if (local === TOMBSTONE) { return undefined; } @@ -196,13 +204,27 @@ export class GeneralDKV extends EventEmitter { } const x = { ...this.kv.get(), ...this.local }; for (const key in this.local) { - if (this.local[key] === TOMBSTONE) { + if ((this.local[key] ?? this.localSaved[key]) === TOMBSTONE) { delete x[key]; } } return x; }; + has = (key: string): boolean => { + if (this.kv == null) { + throw Error("closed"); + } + const a = this.local[key] ?? this.localSaved[key]; + if (a === TOMBSTONE) { + return false; + } + if (a !== undefined) { + return true; + } + return this.kv.has(key); + }; + time = (key?: string) => { if (this.kv == null) { throw Error("closed"); @@ -217,10 +239,27 @@ export class GeneralDKV extends EventEmitter { this.kv.assertValidKey(key); }; - delete = (key) => { - this.assertValidKey(key); + private _delete = (key) => { this.local[key] = TOMBSTONE; this.changed.add(key); + }; + + delete = (key) => { + this.assertValidKey(key); + this._delete(key); + this.save(); + }; + + clear = () => { + if (this.kv == null) { + throw Error("closed"); + } + for (const key in this.kv.get()) { + this._delete(key); + } + for (const key in this.local) { + this._delete(key); + } this.save(); }; @@ -228,12 +267,14 @@ export class GeneralDKV extends EventEmitter { if (args.length == 2) { this.assertValidKey(args[0]); this.local[args[0]] = args[1] ?? TOMBSTONE; + delete this.localSaved[args[0]]; this.changed.add(args[0]); } else { const obj = args[0]; for (const key in obj) { this.assertValidKey(key); this.local[key] = obj[key] ?? TOMBSTONE; + delete this.localSaved[key]; this.changed.add(key); } } @@ -279,6 +320,7 @@ export class GeneralDKV extends EventEmitter { await this.kv.delete(key); delete obj[key]; if (!this.changed.has(key)) { + this.localSaved[key] = this.local[key]; delete this.local[key]; } } @@ -292,12 +334,15 @@ export class GeneralDKV extends EventEmitter { await this.kv.set(key, obj[key]); if (obj[key] === this.local[key] && !this.changed.has(key)) { // successfully saved this + this.localSaved[key] = this.local[key]; delete this.local[key]; } } catch (err) { if (err.code == "REJECT" && err.key) { + const value = this.local[err.key]; delete this.local[err.key]; // can never save this. - this.emit("reject", { key: err.key, value: this.local[err.key] }); + delete this.localSaved[err.key]; + this.emit("reject", { key: err.key, value }); } throw err; } diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index 336bc9af93..d747dceaf7 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -260,6 +260,10 @@ export class GeneralKV extends EventEmitter { } }; + has = (key: string): boolean => { + return this.all?.[key] !== undefined; + }; + time = (key?: string) => { if (key == null) { return this.times; From 3e61607280549c9a5b757e848cff56ef457cbc06 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 11 Feb 2025 19:08:48 +0000 Subject: [PATCH 171/281] nats dkv -- more unit tests --- src/packages/backend/nats/test/dkv.test.ts | 161 ++++++++++++++------- src/packages/nats/sync/dkv.ts | 5 + src/packages/nats/sync/general-dkv.ts | 40 +++-- src/packages/nats/sync/general-kv.ts | 4 + 4 files changed, 138 insertions(+), 72 deletions(-) diff --git a/src/packages/backend/nats/test/dkv.test.ts b/src/packages/backend/nats/test/dkv.test.ts index ba04b50a0c..f8f7938ce1 100644 --- a/src/packages/backend/nats/test/dkv.test.ts +++ b/src/packages/backend/nats/test/dkv.test.ts @@ -1,6 +1,8 @@ import { dkv as createDkv } from "@cocalc/backend/nats/sync"; import { once } from "@cocalc/util/async-utils"; +import { delay } from "awaiting"; + describe("create a public dkv and do basic operations", () => { let kv; const name = `test-${Math.random()}`; @@ -118,16 +120,15 @@ describe("check server assigned times", () => { describe("test deleting and clearing a dkv", () => { let kv1; let kv2; - const name = `test-${Math.random()}`; - const reset = () => { - kv1.clear(); - kv2.clear(); + const reset = async () => { + const name = `test-${Math.random()}`; + kv1 = await createDkv({ name }, { noCache: true }); + kv2 = await createDkv({ name }, { noCache: true }); }; it("creates the dkv twice without caching so can make sure sync works", async () => { - kv1 = await createDkv({ name }, { noCache: true }); - kv2 = await createDkv({ name }, { noCache: true }); + await reset(); expect(kv1.get()).toEqual({}); expect(kv2.get()).toEqual({}); expect(kv1 === kv2).toBe(false); @@ -147,85 +148,143 @@ describe("test deleting and clearing a dkv", () => { }); it("adds an entry, clears it and confirms", async () => { - reset(); + await reset(); - kv1.foo = "bar"; + kv1.foo10 = "bar"; await once(kv2, "change"); - expect(kv2.foo).toBe(kv1.foo); + expect(kv2.foo10).toBe(kv1.foo10); kv2.clear(); - expect(kv2.has("foo")).toBe(false); + expect(kv2.has("foo10")).toBe(false); await once(kv1, "change"); - expect(kv1.has("foo")).toBe(false); + expect(kv1.has("foo10")).toBe(false); }); it("adds an entry, syncs, adds another local entry (not sync'd), clears in sync and confirms NOT everything was cleared", async () => { - reset(); - kv1.foo = Math.random(); + await reset(); + kv1["foo"] = Math.random(); await kv1.save(); - if (kv2.foo != kv1.foo) { + if (kv2["foo"] != kv1["foo"]) { await once(kv2, "change"); } - expect(kv2.foo).toBe(kv1.foo); - kv1.xxx = "yyy"; - expect(kv2.xxx).toBe(undefined); - // this ONLY clears foo, not xxx + expect(kv2["foo"]).toBe(kv1["foo"]); + kv1["bar"] = "yyy"; + expect(kv2["bar"]).toBe(undefined); + // this ONLY clears 'foo', not 'bar' kv2.clear(); await once(kv1, "change"); - expect(kv1.has("xxx")).toBe(true); + expect(kv1.has("bar")).toBe(true); }); it("adds an entry, syncs, adds another local entry (not sync'd), clears in first one, and confirms everything was cleared", async () => { - reset(); + await reset(); - kv1.foo = Math.random(); + const key = Math.random(); + kv1[key] = Math.random(); await kv1.save(); - if (kv2.foo != kv1.foo) { + if (kv2[key] != kv1[key]) { await once(kv2, "change"); } - kv1.xxx = "yyy"; - expect(kv2.xxx).toBe(undefined); + const key2 = Math.random(); + kv1[key2] = "yyy"; + expect(kv2[key2]).toBe(undefined); // this ONLY clears foo, not xxx kv1.clear(); - expect(kv1.has("xxx")).toBe(false); + expect(kv1.has(key2)).toBe(false); }); }); -describe("set several items, confirm exist, save, and confirm they are still there", () => { +describe("set several items, confirm write worked, save, and confirm they are still there after save", () => { const name = `test-${Math.random()}`; - const count = 10; + const count = 100; + // the time thresholds should be trivial for only 100 items it(`adds ${count} entries`, async () => { + const kv = await createDkv({ name }); + expect(kv.get()).toEqual({}); + const obj: any = {}; + const t0 = Date.now(); + for (let i = 0; i < count; i++) { + obj[`${i}`] = i; + kv.set(`${i}`, i); + } + expect(Date.now() - t0).toBeLessThan(50); + expect(Object.keys(kv.get()).length).toEqual(count); + expect(kv.get()).toEqual(obj); + await kv.save(); + expect(Date.now() - t0).toBeLessThan(500); + expect(Object.keys(kv.get()).length).toEqual(count); + // the local state maps should also get cleared quickly, + // but there is no event for this, so we loop: + // @ts-ignore: saved is private + while (Object.keys(kv.generalDKV.saved).length > 0) { + await delay(5); + } + // @ts-ignore: local is private + expect(kv.generalDKV.local).toEqual({}); + // @ts-ignore: saved is private + expect(kv.generalDKV.saved).toEqual({}); + }); +}); + +describe("do an insert and clear test", () => { + const name = `test-${Math.random()}`; + const count = 100; + it(`adds ${count} entries, saves, clears, and confirms empty`, async () => { const kv = await createDkv({ name }); expect(kv.get()).toEqual({}); for (let i = 0; i < count; i++) { kv[`${i}`] = i; } - console.log(kv.get()); expect(Object.keys(kv.get()).length).toEqual(count); await kv.save(); - console.log(kv.get()); expect(Object.keys(kv.get()).length).toEqual(count); + kv.clear(); + expect(kv.get()).toEqual({}); + await kv.save(); + expect(kv.get()).toEqual({}); }); }); -// import { delay } from "awaiting"; - -// describe("do a large insert and clear stress test", () => { -// const name = `test-${Math.random()}`; -// const count = 10; -// it(`adds ${count} entries, saves, clears, and confirms empty`, async () => { -// const kv = await createDkv({ name }); -// expect(kv.get()).toEqual({}); -// for (let i = 0; i < count; i++) { -// kv[`${i}`] = i; -// } -// console.log(kv.get()); -// expect(Object.keys(kv.get()).length).toEqual(count); -// await kv.save(); -// console.log(kv.get()); -// expect(Object.keys(kv.get()).length).toEqual(count); -// kv.clear(); -// expect(kv.get()).toEqual({}); -// await kv.save(); -// expect(kv.get()).toEqual({}); -// }); -// }); +describe("create many distinct clients at once, write to all of them, and see that that results are merged", () => { + const name = `test-${Math.random()}`; + const count = 5; + const clients: any[] = []; + + it(`creates the ${count} clients`, async () => { + for (let i = 0; i < count; i++) { + clients[i] = await createDkv({ name }, { noCache: true }); + } + }); + + // what the combination should be + let combined: any = {}; + it("writes a separate key/value for each client", () => { + for (let i = 0; i < count; i++) { + clients[i].set(`${i}`, i); + combined[`${i}`] = i; + expect(clients[i].get(`${i}`)).toEqual(i); + } + }); + + it("saves and checks that everybody has the combined values", async () => { + for (const kv of clients) { + await kv.save(); + } + let done = false; + let i = 0; + while (!done && i < 50) { + done = true; + i += 1; + for (const client of clients) { + if (client.length != count) { + done = false; + await delay(10); + break; + } + } + } + for (const client of clients) { + expect(client.length).toEqual(count); + expect(client.get()).toEqual(combined); + } + }); +}); diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index e0b9848d95..1bb3bdae4c 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -174,6 +174,11 @@ export class DKV extends EventEmitter { } }; + get length() { + // not efficient? + return Object.keys(this.get()).length; + } + set = (key: string, value: any) => { if (this.generalDKV == null) { throw Error("closed"); diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index 6dc5c01953..96227bc0aa 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -91,11 +91,8 @@ export type MergeFunction = (opts: { export class GeneralDKV extends EventEmitter { private kv?: GeneralKV; private merge?: MergeFunction; - // local values that have NOT been saved to NATS: private local: { [key: string]: any } = {}; - // local values that HAVE been saved to NATS but may not yet be in this.kv.get() - private localSaved: { [key: string]: any } = {}; - // these may have been changed locally: + private saved: { [key: string]: any } = {}; private changed: Set = new Set(); constructor({ @@ -152,15 +149,14 @@ export class GeneralDKV extends EventEmitter { }; private handleRemoteChange = ({ key, value: remote, prev }) => { - // local = value we have NOT saved: - const local = this.local[key] ?? this.localSaved[key]; + const local = this.local[key]; let value: any = remote; if (local !== undefined) { if (isEqual(local, remote)) { // we have a local change, but it's the same change as remote, so just // forget about our local change. delete this.local[key]; - delete this.localSaved[key]; + delete this.saved[key]; } else { try { value = this.merge?.({ key, local, remote, prev }) ?? local; @@ -179,11 +175,10 @@ export class GeneralDKV extends EventEmitter { if (isEqual(value, remote)) { // no change, so forget our local value delete this.local[key]; - delete this.localSaved[key]; + delete this.saved[key]; } else { // resolve with the new value, or if it is undefined, a TOMBSTONE, meaning choice is to delete. this.local[key] = value ?? TOMBSTONE; - delete this.localSaved[key]; } } } @@ -196,7 +191,7 @@ export class GeneralDKV extends EventEmitter { } if (key != null) { this.assertValidKey(key); - const local = this.local[key] ?? this.localSaved[key]; + const local = this.local[key]; if (local === TOMBSTONE) { return undefined; } @@ -204,18 +199,23 @@ export class GeneralDKV extends EventEmitter { } const x = { ...this.kv.get(), ...this.local }; for (const key in this.local) { - if ((this.local[key] ?? this.localSaved[key]) === TOMBSTONE) { + if (this.local[key] === TOMBSTONE) { delete x[key]; } } return x; }; + get length() { + // not efficient + return Object.keys(this.get()).length; + } + has = (key: string): boolean => { if (this.kv == null) { throw Error("closed"); } - const a = this.local[key] ?? this.localSaved[key]; + const a = this.local[key]; if (a === TOMBSTONE) { return false; } @@ -267,14 +267,12 @@ export class GeneralDKV extends EventEmitter { if (args.length == 2) { this.assertValidKey(args[0]); this.local[args[0]] = args[1] ?? TOMBSTONE; - delete this.localSaved[args[0]]; this.changed.add(args[0]); } else { const obj = args[0]; for (const key in obj) { this.assertValidKey(key); this.local[key] = obj[key] ?? TOMBSTONE; - delete this.localSaved[key]; this.changed.add(key); } } @@ -285,11 +283,13 @@ export class GeneralDKV extends EventEmitter { if (this.kv == null) { return false; } - return this.changed.size > 0 || Object.keys(this.local).length > 0; + return this.unsavedChanges().length > 0; }; unsavedChanges = () => { - return Object.keys(this.local); + return Object.keys(this.local).filter( + (key) => this.local[key] !== this.saved[key], + ); }; save = reuseInFlight(async () => { @@ -320,8 +320,7 @@ export class GeneralDKV extends EventEmitter { await this.kv.delete(key); delete obj[key]; if (!this.changed.has(key)) { - this.localSaved[key] = this.local[key]; - delete this.local[key]; + this.saved[key] = this.local[key]; } } } @@ -334,14 +333,13 @@ export class GeneralDKV extends EventEmitter { await this.kv.set(key, obj[key]); if (obj[key] === this.local[key] && !this.changed.has(key)) { // successfully saved this - this.localSaved[key] = this.local[key]; - delete this.local[key]; + this.saved[key] = this.local[key]; } } catch (err) { if (err.code == "REJECT" && err.key) { const value = this.local[err.key]; delete this.local[err.key]; // can never save this. - delete this.localSaved[err.key]; + delete this.saved[err.key]; // can never save this. this.emit("reject", { key: err.key, value }); } throw err; diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index d747dceaf7..442df3c949 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -260,6 +260,10 @@ export class GeneralKV extends EventEmitter { } }; + get length() { + return Object.keys(this.all ?? {}).length; + } + has = (key: string): boolean => { return this.all?.[key] !== undefined; }; From 3e080254b56801bc104b7a59d166e31140225f28 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 11 Feb 2025 20:55:33 +0000 Subject: [PATCH 172/281] nats: unit testing 3-way merge conflict implementation --- .../backend/nats/test/dkv-merge.test.ts | 121 ++++++++++++++++++ src/packages/backend/nats/test/dkv.test.ts | 1 - src/packages/nats/sync/dkv.ts | 39 +++++- src/packages/nats/sync/general-dkv.ts | 43 +++++-- 4 files changed, 194 insertions(+), 10 deletions(-) create mode 100644 src/packages/backend/nats/test/dkv-merge.test.ts diff --git a/src/packages/backend/nats/test/dkv-merge.test.ts b/src/packages/backend/nats/test/dkv-merge.test.ts new file mode 100644 index 0000000000..1ae7c8090c --- /dev/null +++ b/src/packages/backend/nats/test/dkv-merge.test.ts @@ -0,0 +1,121 @@ +// Testing merge conflicts with dkv +// pnpm exec jest --watch --forceExit --detectOpenHandles "dkv-merge.test.ts" + +import { dkv as createDkv } from "@cocalc/backend/nats/sync"; +import { once } from "@cocalc/util/async-utils"; +//import { delay } from "awaiting"; + +async function getKvs(opts?) { + const name = `test-${Math.random()}`; + // We disable autosave so that we have more precise control of how conflicts + // get resolved, etc. for testing purposes. + const kv1 = await createDkv( + { name, noAutosave: true, ...opts }, + { noCache: true }, + ); + const kv2 = await createDkv( + { name, noAutosave: true, ...opts }, + { noCache: true }, + ); + return { kv1, kv2 }; +} + +describe("test the default 'local first' merge conflict resolution function", () => { + it("sets up and resolves a merge conflict", async () => { + const { kv1, kv2 } = await getKvs(); + kv1.set("x", 5); + kv2.set("x", 10); + expect(kv1["x"]).toEqual(5); + expect(kv2["x"]).toEqual(10); + + // now make kv2 save, which makes kv1 detect a conflict: + await kv2.save(); + // kv1 just resolves it in its own favor. + expect(kv1["x"]).toEqual(5); + await kv1.save(); + + // wait until kv2 gets a change, which will be learning about + // how the merge conflict got resolved. + if (kv2.get("x") != 5) { + // might have to wait + await once(kv2, "change"); + } + expect(kv2["x"]).toEqual(5); + }); +}); + +describe("test the default 'local first' merge conflict resolution function, but where we do the sets in the opposite order", () => { + it("sets up and resolves a merge conflict", async () => { + const { kv1, kv2 } = await getKvs(); + kv2.set("x", 10); + kv1.set("x", 5); + expect(kv1["x"]).toEqual(5); + expect(kv2["x"]).toEqual(10); + + // now make kv2 save, which makes kv1 detect a conflict: + await kv2.save(); + // kv1 just resolves it in its own favor. + expect(kv1["x"]).toEqual(5); + await kv1.save(); + + // wait until kv2 gets a change, which will be learning about + // how the merge conflict got resolved. + if (kv2.get("x") != 5) { + // might have to wait + await once(kv2, "change"); + } + expect(kv2["x"]).toEqual(5); + }); +}); + +describe("test a trivial merge conflict resolution function", () => { + it("sets up and resolves a merge conflict", async () => { + const { kv1, kv2 } = await getKvs({ + merge: () => { + // our merge strategy is to replace the value by 'conflict' + return "conflict"; + }, + }); + kv1.set("x", 5); + kv2.set("x", 10); + expect(kv1["x"]).toEqual(5); + expect(kv2["x"]).toEqual(10); + + // now make kv2 save, which makes kv1 detect a conflict: + await kv2.save(); + if (kv1["x"] != "conflict") { + // might have to wait + await once(kv1, "change"); + } + expect(kv1["x"]).toEqual("conflict"); + + await kv1.save(); + // wait until kv2 gets a change, which will be learning about + // how the merge conflict got resolved. + if (kv2["x"] != "conflict") { + // might have to wait + await once(kv2, "change"); + } + expect(kv2["x"]).toEqual("conflict"); + }); +}); + +describe("test a sophisticated 3-way merge of strings conflict resolution function", () => { + it("sets up and resolves a merge conflict", async () => { + const { kv1, kv2 } = await getKvs({ + merge: ({ local, remote, prev = "" }) => { + // our merge strategy is to replace the value by 'conflict' + return `${local}${remote}${prev}`; + }, + }); + kv1.set("x", 'cocalc'); + await kv1.save(); + if (kv2["x"] != "cocalc") { + // might have to wait + await once(kv2, "change"); + } + expect(kv2["x"]).toEqual("cocalc"); + + + }); +}); diff --git a/src/packages/backend/nats/test/dkv.test.ts b/src/packages/backend/nats/test/dkv.test.ts index f8f7938ce1..7443b86725 100644 --- a/src/packages/backend/nats/test/dkv.test.ts +++ b/src/packages/backend/nats/test/dkv.test.ts @@ -1,6 +1,5 @@ import { dkv as createDkv } from "@cocalc/backend/nats/sync"; import { once } from "@cocalc/util/async-utils"; - import { delay } from "awaiting"; describe("create a public dkv and do basic operations", () => { diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index 1bb3bdae4c..6fb239308c 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -24,6 +24,7 @@ import { sha1 } from "@cocalc/util/misc"; export interface DKVOptions extends KVOptions { merge: MergeFunction; + noAutosave?: boolean; } export class DKV extends EventEmitter { @@ -39,6 +40,7 @@ export class DKV extends EventEmitter { project_id, merge, env, + noAutosave, limits, }: DKVOptions) { super(); @@ -55,6 +57,7 @@ export class DKV extends EventEmitter { filter: `${this.prefix}.>`, env, merge, + noAutosave, limits, }; @@ -86,7 +89,41 @@ export class DKV extends EventEmitter { if (this.generalDKV != null) { return; } - this.generalDKV = new GeneralDKV(this.opts); + // the merge conflict algorithm must be adapted since we encode + // keys and values specially in this class. + const merge = (opts) => { + // here is what the input might look like: + // opts = { + // key: '71d7616250fed4dc27b70ee3b934178a3b196bbb.11f6ad8ec52a2984abaafd7c3b516503785c2072', + // remote: { key: 'x', value: 10 }, + // local: { key: 'x', value: 5 }, + // prev: { key: 'x', value: 3 } + // } + const key = opts.local?.key; + if (key == null) { + console.warn("BUG in merge conflict resolution", opts); + throw Error("local key must be defined"); + } + const local = opts.local.value; + const remote = opts.remote?.value; + const prev = opts.prev?.value; + + let value; + try { + value = this.opts.merge?.({ key, local, remote, prev }) ?? local; + } catch (err) { + console.warn("exception in merge conflict resolution", err); + value = local; + } + // console.log( + // "conflict resolution: ", + // { key, local, remote, prev }, + // "-->", + // { value }, + // ); + return { key, value }; + }; + this.generalDKV = new GeneralDKV({ ...this.opts, merge }); this.generalDKV.on("change", ({ value, prev }) => { if (value != null && value !== TOMBSTONE) { this.emit("change", { diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index 96227bc0aa..ca62165682 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -94,6 +94,7 @@ export class GeneralDKV extends EventEmitter { private local: { [key: string]: any } = {}; private saved: { [key: string]: any } = {}; private changed: Set = new Set(); + private noAutosave: boolean; constructor({ name, @@ -101,6 +102,7 @@ export class GeneralDKV extends EventEmitter { filter, merge, options, + noAutosave, limits, }: { name: string; @@ -115,11 +117,16 @@ export class GeneralDKV extends EventEmitter { // filter: optionally restrict to subset of named kv store matching these subjects. // NOTE: any key name that you *set or delete* must match one of these filter: string | string[]; - options?; limits?: KVLimits; + // if noAutosave is set, local changes are never saved until you explicitly + // call "await this.save()", which will try once to save. New changes may + // not be saved though. + noAutosave?: boolean; + options?; }) { super(); this.merge = merge; + this.noAutosave = !!noAutosave; //this.limits = limits; this.kv = new GeneralKV({ name, env, filter, options, limits }); } @@ -149,7 +156,7 @@ export class GeneralDKV extends EventEmitter { }; private handleRemoteChange = ({ key, value: remote, prev }) => { - const local = this.local[key]; + const local = this.local[key] === TOMBSTONE ? undefined : this.local[key]; let value: any = remote; if (local !== undefined) { if (isEqual(local, remote)) { @@ -158,8 +165,10 @@ export class GeneralDKV extends EventEmitter { delete this.local[key]; delete this.saved[key]; } else { + //console.log("merge conflict", { key, remote, local, prev }); try { value = this.merge?.({ key, local, remote, prev }) ?? local; + // console.log("merge conflict --> ", value); // console.log("handle merge conflict", { // key, // local, @@ -167,18 +176,26 @@ export class GeneralDKV extends EventEmitter { // prev, // value, // }); - } catch { + } catch (err) { + console.warn("exception in merge conflict resolution", err); // user provided a merge function that throws an exception. We select local, since // it is the newest, i.e., "last write wins" value = local; + // console.log("merge conflict ERROR --> ", err, value); } if (isEqual(value, remote)) { // no change, so forget our local value delete this.local[key]; delete this.saved[key]; } else { - // resolve with the new value, or if it is undefined, a TOMBSTONE, meaning choice is to delete. - this.local[key] = value ?? TOMBSTONE; + // resolve with the new value, or if it is undefined, a TOMBSTONE, + // meaning choice is to delete. + // console.log("conflict resolution: ", { key, value }); + if (value === TOMBSTONE) { + this.delete(key); + } else { + this.set(key, value); + } } } } @@ -247,7 +264,9 @@ export class GeneralDKV extends EventEmitter { delete = (key) => { this.assertValidKey(key); this._delete(key); - this.save(); + if (!this.noAutosave) { + this.save(); + } }; clear = () => { @@ -260,7 +279,9 @@ export class GeneralDKV extends EventEmitter { for (const key in this.local) { this._delete(key); } - this.save(); + if (!this.noAutosave) { + this.save(); + } }; set = (...args) => { @@ -276,7 +297,9 @@ export class GeneralDKV extends EventEmitter { this.changed.add(key); } } - this.save(); + if (!this.noAutosave) { + this.save(); + } }; hasUnsavedChanges = () => { @@ -293,6 +316,10 @@ export class GeneralDKV extends EventEmitter { }; save = reuseInFlight(async () => { + if (this.noAutosave) { + await this.attemptToSave(); + return; + } let d = 100; while (true) { try { From 2e1b55621712fe7d100df9778edd96e56925cd55 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 11 Feb 2025 22:24:42 +0000 Subject: [PATCH 173/281] nats kv -- more unit tests, especially involving 3-way merge --- .../backend/nats/test/dkv-merge.test.ts | 58 +++++++++++++++++-- src/packages/nats/sync/dkv.ts | 2 +- src/packages/nats/sync/general-dkv.ts | 25 ++++++-- 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/src/packages/backend/nats/test/dkv-merge.test.ts b/src/packages/backend/nats/test/dkv-merge.test.ts index 1ae7c8090c..1f99356413 100644 --- a/src/packages/backend/nats/test/dkv-merge.test.ts +++ b/src/packages/backend/nats/test/dkv-merge.test.ts @@ -3,7 +3,7 @@ import { dkv as createDkv } from "@cocalc/backend/nats/sync"; import { once } from "@cocalc/util/async-utils"; -//import { delay } from "awaiting"; +import { diff_match_patch } from "@cocalc/util/dmp"; async function getKvs(opts?) { const name = `test-${Math.random()}`; @@ -100,22 +100,72 @@ describe("test a trivial merge conflict resolution function", () => { }); }); -describe("test a sophisticated 3-way merge of strings conflict resolution function", () => { +describe("test a 3-way merge of strings conflict resolution function", () => { + const dmp = new diff_match_patch(); + const threeWayMerge = (opts: { + prev: string; + local: string; + remote: string; + }) => { + return dmp.patch_apply( + dmp.patch_make(opts.prev, opts.local), + opts.remote, + )[0]; + }; it("sets up and resolves a merge conflict", async () => { const { kv1, kv2 } = await getKvs({ merge: ({ local, remote, prev = "" }) => { // our merge strategy is to replace the value by 'conflict' - return `${local}${remote}${prev}`; + return threeWayMerge({ local, remote, prev }); }, }); - kv1.set("x", 'cocalc'); + kv1.set("x", "cocalc"); await kv1.save(); if (kv2["x"] != "cocalc") { // might have to wait await once(kv2, "change"); } expect(kv2["x"]).toEqual("cocalc"); + await kv2.save(); + kv2.set("x", "cocalc!"); + kv1.set("x", "LOVE cocalc"); + await kv2.save(); + if (kv1.get("x") != "LOVE cocalc!") { + await once(kv1, "change"); + } + expect(kv1.get("x")).toEqual("LOVE cocalc!"); + await kv1.save(); + if (kv2.get("x") != "LOVE cocalc!") { + await once(kv2, "change"); + } + expect(kv2.get("x")).toEqual("LOVE cocalc!"); + }); +}); +describe("test a 3-way merge of that merges objects", () => { + it("sets up and resolves a merge conflict", async () => { + const { kv1, kv2 } = await getKvs({ + merge: ({ local, remote }) => { + return { ...remote, ...local }; + }, + }); + kv1.set("x", { a: 5 }); + await kv1.save(); + await once(kv2, "change"); + expect(kv2["x"]).toEqual({ a: 5 }); + + kv1.set("x", { a: 5, b: 15, c: 12 }); + kv2.set("x", { a: 5, b: 7, d: 3 }); + await kv2.save(); + if (kv1.get("x").d != 3) { + await once(kv1, "change"); + } + expect(kv1.get("x")).toEqual({ a: 5, b: 15, c: 12, d: 3 }); + await kv1.save(); + if (kv2.get("x").b != 15) { + await once(kv2, "change"); + } + expect(kv2.get("x")).toEqual({ a: 5, b: 15, c: 12, d: 3 }); }); }); diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index 6fb239308c..1c86e4a78d 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -239,7 +239,7 @@ export class DKV extends EventEmitter { }; save = async () => { - await this.generalDKV?.save(); + return await this.generalDKV?.save(); }; } diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index ca62165682..56748ddf10 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -317,13 +317,21 @@ export class GeneralDKV extends EventEmitter { save = reuseInFlight(async () => { if (this.noAutosave) { - await this.attemptToSave(); - return; + return await this.attemptToSave(); + // one example error when there's a conflict brewing: + /* + { + code: 10071, + name: 'JetStreamApiError', + message: 'wrong last sequence: 84492' + } + */ } let d = 100; while (true) { + let status; try { - await this.attemptToSave(); + status = await this.attemptToSave(); //console.log("successfully saved"); } catch { d = Math.min(10000, d * 1.3) + Math.random() * 100; @@ -331,7 +339,7 @@ export class GeneralDKV extends EventEmitter { // console.log("temporary issue saving") } if (!this.hasUnsavedChanges()) { - return; + return status; } } }); @@ -341,10 +349,14 @@ export class GeneralDKV extends EventEmitter { throw Error("closed"); } this.changed.clear(); + const status = { unsaved: 0, set: 0, delete: 0 }; const obj = { ...this.local }; for (const key in obj) { if (obj[key] === TOMBSTONE) { + status.unsaved += 1; await this.kv.delete(key); + status.delete += 1; + status.unsaved -= 1; delete obj[key]; if (!this.changed.has(key)) { this.saved[key] = this.local[key]; @@ -357,7 +369,10 @@ export class GeneralDKV extends EventEmitter { return; } try { + status.unsaved += 1; await this.kv.set(key, obj[key]); + status.unsaved -= 1; + status.set += 1; if (obj[key] === this.local[key] && !this.changed.has(key)) { // successfully saved this this.saved[key] = this.local[key]; @@ -367,11 +382,13 @@ export class GeneralDKV extends EventEmitter { const value = this.local[err.key]; delete this.local[err.key]; // can never save this. delete this.saved[err.key]; // can never save this. + status.unsaved -= 1; this.emit("reject", { key: err.key, value }); } throw err; } }; await awaitMap(Object.keys(obj), MAX_PARALLEL, f); + return status; }); } From e124221fd09d4ea62219a47b4f755a7e4fbf75f9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 12 Feb 2025 01:41:26 +0000 Subject: [PATCH 174/281] nats: fix some bugs revealed by testing and caused by testing --- .../backend/nats/test/dkv-merge.test.ts | 8 +++-- src/packages/backend/nats/test/dkv.test.ts | 24 ++++++++----- src/packages/nats/sync/general-dkv.ts | 34 +++++++++++++------ 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/src/packages/backend/nats/test/dkv-merge.test.ts b/src/packages/backend/nats/test/dkv-merge.test.ts index 1f99356413..d99c86ef6a 100644 --- a/src/packages/backend/nats/test/dkv-merge.test.ts +++ b/src/packages/backend/nats/test/dkv-merge.test.ts @@ -1,5 +1,9 @@ -// Testing merge conflicts with dkv -// pnpm exec jest --watch --forceExit --detectOpenHandles "dkv-merge.test.ts" +/* +Testing merge conflicts with dkv + +DEVELOPMENT: +pnpm exec jest --watch --forceExit --detectOpenHandles "dkv-merge.test.ts" +*/ import { dkv as createDkv } from "@cocalc/backend/nats/sync"; import { once } from "@cocalc/util/async-utils"; diff --git a/src/packages/backend/nats/test/dkv.test.ts b/src/packages/backend/nats/test/dkv.test.ts index 7443b86725..7337f1ecb1 100644 --- a/src/packages/backend/nats/test/dkv.test.ts +++ b/src/packages/backend/nats/test/dkv.test.ts @@ -1,3 +1,11 @@ +/* +Testing basic ops with dkv + +DEVELOPMENT: +pnpm exec jest --watch --forceExit --detectOpenHandles "dkv.test.ts" + +*/ + import { dkv as createDkv } from "@cocalc/backend/nats/sync"; import { once } from "@cocalc/util/async-utils"; import { delay } from "awaiting"; @@ -194,8 +202,8 @@ describe("test deleting and clearing a dkv", () => { describe("set several items, confirm write worked, save, and confirm they are still there after save", () => { const name = `test-${Math.random()}`; - const count = 100; - // the time thresholds should be trivial for only 100 items + const count = 50; + // the time thresholds should be "trivial" it(`adds ${count} entries`, async () => { const kv = await createDkv({ name }); expect(kv.get()).toEqual({}); @@ -209,13 +217,13 @@ describe("set several items, confirm write worked, save, and confirm they are st expect(Object.keys(kv.get()).length).toEqual(count); expect(kv.get()).toEqual(obj); await kv.save(); - expect(Date.now() - t0).toBeLessThan(500); + expect(Date.now() - t0).toBeLessThan(1000); expect(Object.keys(kv.get()).length).toEqual(count); - // the local state maps should also get cleared quickly, - // but there is no event for this, so we loop: - // @ts-ignore: saved is private - while (Object.keys(kv.generalDKV.saved).length > 0) { - await delay(5); + // // the local state maps should also get cleared quickly, + // // but there is no event for this, so we loop: + // @ts-ignore: saved is private + while (Object.keys(kv.generalDKV.local).length > 0) { + await delay(10); } // @ts-ignore: local is private expect(kv.generalDKV.local).toEqual({}); diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index 56748ddf10..d449f1e5eb 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -90,6 +90,7 @@ export type MergeFunction = (opts: { export class GeneralDKV extends EventEmitter { private kv?: GeneralKV; + private jc?; private merge?: MergeFunction; private local: { [key: string]: any } = {}; private saved: { [key: string]: any } = {}; @@ -119,8 +120,8 @@ export class GeneralDKV extends EventEmitter { filter: string | string[]; limits?: KVLimits; // if noAutosave is set, local changes are never saved until you explicitly - // call "await this.save()", which will try once to save. New changes may - // not be saved though. + // call "await this.save()", which will try once to save. Changes made during + // the save may not be saved though. noAutosave?: boolean; options?; }) { @@ -128,6 +129,7 @@ export class GeneralDKV extends EventEmitter { this.merge = merge; this.noAutosave = !!noAutosave; //this.limits = limits; + this.jc = env.jc; this.kv = new GeneralKV({ name, env, filter, options, limits }); } @@ -165,7 +167,7 @@ export class GeneralDKV extends EventEmitter { delete this.local[key]; delete this.saved[key]; } else { - //console.log("merge conflict", { key, remote, local, prev }); + // console.log("merge conflict", { key, remote, local, prev }); try { value = this.merge?.({ key, local, remote, prev }) ?? local; // console.log("merge conflict --> ", value); @@ -284,16 +286,28 @@ export class GeneralDKV extends EventEmitter { } }; + private toValue = (obj) => { + if (obj === undefined) { + return TOMBSTONE; + } + // It's EXTREMELY important that anything we save to NATS has the property that + // jc.decode(jc.encode(obj)) is the identity map. That is very much NOT + // the case for stuff that set gets called on, e.g., {a:new Date()}. + // Thus before storing it in in any way, we ensure this immediately: + return this.jc.decode(this.jc.encode(obj)); + }; + set = (...args) => { if (args.length == 2) { this.assertValidKey(args[0]); - this.local[args[0]] = args[1] ?? TOMBSTONE; + const obj = this.toValue(args[1]); + this.local[args[0]] = obj; this.changed.add(args[0]); } else { const obj = args[0]; for (const key in obj) { this.assertValidKey(key); - this.local[key] = obj[key] ?? TOMBSTONE; + this.local[key] = this.toValue(obj[key]); this.changed.add(key); } } @@ -333,14 +347,14 @@ export class GeneralDKV extends EventEmitter { try { status = await this.attemptToSave(); //console.log("successfully saved"); - } catch { - d = Math.min(10000, d * 1.3) + Math.random() * 100; - await delay(d); - // console.log("temporary issue saving") + } catch (_err) { + //console.log("temporary issue saving", this.kv?.name, _err); } if (!this.hasUnsavedChanges()) { return status; } + d = Math.min(10000, d * 1.3) + Math.random() * 100; + await delay(d); } }); @@ -373,7 +387,7 @@ export class GeneralDKV extends EventEmitter { await this.kv.set(key, obj[key]); status.unsaved -= 1; status.set += 1; - if (obj[key] === this.local[key] && !this.changed.has(key)) { + if (!this.changed.has(key)) { // successfully saved this this.saved[key] = this.local[key]; } From b553711de7fb60d1c4a3fe4afeb32ab741b4e6f8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 12 Feb 2025 02:45:30 +0000 Subject: [PATCH 175/281] nats: fix major issue with hangs at scale (due to lack of auth) - NATS is frickin' insane where you just have to randomly test and guess to get things to work at scale. SOOOO HARD TO USE! --- src/packages/database/nats/changefeeds.ts | 1 + src/packages/nats/util.ts | 10 +++++----- src/packages/server/nats/auth.ts | 4 ++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/packages/database/nats/changefeeds.ts b/src/packages/database/nats/changefeeds.ts index 7084ffbfcb..64630be42d 100644 --- a/src/packages/database/nats/changefeeds.ts +++ b/src/packages/database/nats/changefeeds.ts @@ -186,6 +186,7 @@ const createChangefeed = reuseInFlight( } for (const key in current) { if (!databaseKeys.has(key)) { + console.log("remove from synctable", key); synctable.delete(key); } } diff --git a/src/packages/nats/util.ts b/src/packages/nats/util.ts index 50b07e3377..88d07c38e4 100644 --- a/src/packages/nats/util.ts +++ b/src/packages/nats/util.ts @@ -28,17 +28,17 @@ export async function getAllFromKv({ const times: { [key: string]: Date } = {}; const watch = await kv.watch({ key, ignoreDeletes: false }); if (watch._data._info.num_pending > 0) { - for await (const { key, value, revision, sm } of watch) { + for await (const { key: key0, value, revision, sm } of watch) { if (value.length > 0) { // we MUST check value.length because we do NOT ignoreDeletes. // we do NOT ignore deletes so that sm.di.pending goes down to 0. // Otherwise, there is no way in general to know when we are done. - all[key] = value; - revisions[key] = revision; - times[key] = sm.time; + all[key0] = value; + revisions[key0] = revision; + times[key0] = sm.time; } - // console.log("getAllFromKv", key, sm.di.pending); if (sm.di.pending <= 0) { + // **NOTE! This will hang and never get hit if you don't have the $JC.FC.... auth enabled!!!!** break; } } diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index e4be78f8d5..84966333c8 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -138,6 +138,8 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { // the account-specific kv stores goalPub.add(`$JS.API.*.*.KV_account-${userId}`); goalPub.add(`$JS.API.*.*.KV_account-${userId}.>`); + // this FC is needed for "flow control" - without this, you get random hangs forever at scale! + goalPub.add(`$JS.FC.KV_account-${userId}.>`); const pool = getPool(); // all RUNNING projects with the user's group @@ -351,6 +353,8 @@ function projectSubjects(project_id: string) { // The unique project-wide jetstream key:value store pub.add(`$JS.*.*.*.KV_project-${project_id}`); pub.add(`$JS.*.*.*.KV_project-${project_id}.>`); + // this FC is needed for "flow control" - without this, you get random hangs forever at scale! + pub.add(`$JS.FC.KV_project-${project_id}.>`); // The unique project-wide jetstream stream: pub.add(`$JS.*.*.*.project-${project_id}`); From 00a37371b8ac012060f4dc25d70f5de9073d4d07 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 12 Feb 2025 05:14:36 +0000 Subject: [PATCH 176/281] nats database synctable -- free synctables (avoid eventemitter leaks, etc.) --- src/packages/database/nats/changefeeds.ts | 13 +++++++++++-- src/packages/frontend/messages/redux.ts | 1 - src/packages/frontend/project_store.ts | 4 ++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/packages/database/nats/changefeeds.ts b/src/packages/database/nats/changefeeds.ts index 64630be42d..987937ddc5 100644 --- a/src/packages/database/nats/changefeeds.ts +++ b/src/packages/database/nats/changefeeds.ts @@ -107,8 +107,9 @@ function queryTable(query) { return Object.keys(query)[0]; } -const changefeedInterest: { [hash: string]: number } = {}; const changefeedHashes: { [id: string]: string } = {}; +const changefeedInterest: { [hash: string]: number } = {}; +const changefeedSynctables: { [hash: string]: any } = {}; function cancelChangefeed(id) { logger.debug("cancelChangefeed", { id }); @@ -117,6 +118,8 @@ function cancelChangefeed(id) { // already canceled return; } + changefeedSynctables[hash]?.close(); + delete changefeedSynctables[hash]; delete changefeedInterest[hash]; delete changefeedHashes[id]; db().user_query_cancel_changefeed({ id }); @@ -165,8 +168,13 @@ const createChangefeed = reuseInFlight( atomic: true, immutable: false, }); + changefeedSynctables[hash] = synctable; - await synctable.init(); + try { + await synctable.init(); + } catch (err) { + cancelChangefeed(changes); + } // if (global.z == null) { // global.z = {}; @@ -245,6 +253,7 @@ const createChangefeed = reuseInFlight( query, ); cancelChangefeed(changes); + return; } } }; diff --git a/src/packages/frontend/messages/redux.ts b/src/packages/frontend/messages/redux.ts index 92a0a53269..e92765d9d2 100644 --- a/src/packages/frontend/messages/redux.ts +++ b/src/packages/frontend/messages/redux.ts @@ -138,7 +138,6 @@ export class MessagesActions extends Actions { }; handleTableUpdate = (messages) => { - const store = this.getStore(); messages = getNotExpired(messages); const threads = getThreads(messages); this.setState({ messages, threads }); diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index aa1f814246..b4f223ccd4 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -594,6 +594,10 @@ export class ProjectStore extends Store { } } const all_paths = deleted_file_variations(path); + const open_files = this.get("open_files"); + if (open_files == null) { + return; + } for (const file of this.get("open_files").keys()) { if (all_paths.indexOf(file) != -1 || misc.startswith(file, path + "/")) { if (!this.has_file_been_viewed(file)) { From 01b5fc64423fc479bf659e60559aac1e0289b884 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 12 Feb 2025 13:56:02 +0000 Subject: [PATCH 177/281] nats: solve the req/reply "inbox security" issue for users via inbox prefix configuration --- src/packages/backend/nats/index.ts | 2 ++ src/packages/database/nats/changefeeds.ts | 2 +- src/packages/frontend/nats/client.ts | 3 ++- src/packages/nats/names.ts | 31 +++++++++++++++++++++++ src/packages/project/nats/connection.ts | 5 ++++ src/packages/server/nats/auth.ts | 22 ++++++++++++---- 6 files changed, 58 insertions(+), 7 deletions(-) diff --git a/src/packages/backend/nats/index.ts b/src/packages/backend/nats/index.ts index 8c1bec851e..9a78b013c1 100644 --- a/src/packages/backend/nats/index.ts +++ b/src/packages/backend/nats/index.ts @@ -7,6 +7,7 @@ export { getEnv } from "./env"; import { delay } from "awaiting"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { CONNECT_OPTIONS } from "@cocalc/util/nats"; +import { inboxPrefix } from "@cocalc/nats/names"; const logger = getLogger("backend:nats"); @@ -33,6 +34,7 @@ export const getConnection = reuseInFlight(async () => { nc = await connect({ ...CONNECT_OPTIONS, authenticator: credsAuthenticator(new TextEncoder().encode(creds)), + inboxPrefix: inboxPrefix({}), }); logger.debug(`connected to ${nc.getServer()}`); } catch (err) { diff --git a/src/packages/database/nats/changefeeds.ts b/src/packages/database/nats/changefeeds.ts index 987937ddc5..7a5a656426 100644 --- a/src/packages/database/nats/changefeeds.ts +++ b/src/packages/database/nats/changefeeds.ts @@ -194,7 +194,7 @@ const createChangefeed = reuseInFlight( } for (const key in current) { if (!databaseKeys.has(key)) { - console.log("remove from synctable", key); + // console.log("remove from synctable", key); synctable.delete(key); } } diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 7859e4823e..fb597a66b5 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -5,7 +5,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { join } from "path"; import * as jetstream from "@nats-io/jetstream"; import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; -import { randomId } from "@cocalc/nats/names"; +import { inboxPrefix, randomId } from "@cocalc/nats/names"; import { browserSubject, projectSubject } from "@cocalc/nats/names"; import { parse_query } from "@cocalc/sync/table/util"; import { sha1 } from "@cocalc/util/misc"; @@ -70,6 +70,7 @@ export class NatsClient { const options = { ...CONNECT_OPTIONS, servers: [server], + inboxPrefix: inboxPrefix({ account_id: this.client.account_id }), }; try { this.nc = await nats.connect(options); diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts index 9b0c1f1e14..153831ba7a 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/nats/names.ts @@ -41,6 +41,37 @@ export function jsName({ return `account-${account_id}`; } +/* +Custom inbox prefix per "user"! + +So can receive response to requests, and that you can ONLY receive responses +to your own messages and nobody else's! This must be used in conjunction with +the inboxPrefix client option when connecting. Note that the NATS docs + https://docs.nats.io/running-a-nats-service/configuration/securing_nats/authorization +do not explain this, instead just emphasizing you're screwed but not giving +the solution, which is very disconcerting! There are a couple of places in +our code where we create connections, and these all must be aware of the +inbox prefix we use. + +This is explained in this natsbyexample page: + +https://natsbyexample.com/examples/auth/private-inbox/cli +*/ +export function inboxPrefix({ + account_id, + project_id, +}: { + account_id?: string; + project_id?: string; +}) { + if (!account_id && !project_id) { + // the hubs + return "_INBOX.hub"; + } + // a project or account: + return `_INBOX.${jsName({ account_id, project_id })}`; +} + export function streamSubject({ project_id, account_id, diff --git a/src/packages/project/nats/connection.ts b/src/packages/project/nats/connection.ts index d1b58c3c6d..c5095b41cb 100644 --- a/src/packages/project/nats/connection.ts +++ b/src/packages/project/nats/connection.ts @@ -1,6 +1,8 @@ import { getLogger } from "@cocalc/project/logger"; import { connect, jwtAuthenticator } from "nats"; import { CONNECT_OPTIONS } from "@cocalc/util/nats"; +import { inboxPrefix as getInboxPrefix } from "@cocalc/nats/names"; +import { project_id } from "@cocalc/project/data"; const logger = getLogger("project:nats:connection"); @@ -11,9 +13,12 @@ export default async function getConnection() { if (!process.env.COCALC_NATS_JWT) { throw Error("environment variable COCALC_NATS_JWT *must* be set"); } + const inboxPrefix = getInboxPrefix({ project_id }); + logger.debug("Using ", { inboxPrefix }); nc = await connect({ ...CONNECT_OPTIONS, authenticator: jwtAuthenticator(process.env.COCALC_NATS_JWT), + inboxPrefix, }); logger.debug(`connected to ${nc.getServer()}`); } diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 84966333c8..a930dc51a0 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -30,6 +30,7 @@ import { natsAccountName } from "@cocalc/backend/nats/conf"; import { throttle } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import isCollaborator from "@cocalc/server/projects/is-collaborator"; +import { inboxPrefix } from "@cocalc/nats/names"; const logger = getLogger("server:nats:auth"); @@ -43,7 +44,7 @@ export async function nsc( // TODO: consider making the names shorter strings using https://www.npmjs.com/package/short-uuid -// A CoCalc User is (so far): a project or an account. +// A CoCalc User is (so far): a project or account or a hub (not covered here). type CoCalcUser = | { account_id: string; @@ -111,13 +112,13 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { const userType = getCoCalcUserType(cocalcUser); // TODO: jetstream permissions are WAY TO BROAD. const goalPub = new Set([ - "_INBOX.>", // so can use request/response + // "_INBOX.>", // !!!! TODO: so can create responses to requests -- fix this it is horribly insecure!!!!! `hub.${userType}.${userId}.>`, // can talk as *only this user* to the hub's api's "$JS.API.INFO", ]); const goalSub = new Set([ - "_INBOX.>", // so can user request/response - "system.>", // access to READ the system info kv store. + inboxPrefix(cocalcUser) + ".>", + "public.>", // access to READ the public system info kv store. ]); // the public jetstream: this makes it available *read only* to all accounts and projects. @@ -204,7 +205,18 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { } } } - const args = ["edit", "signing-key", "--sk", name]; + // We edit the signing key rather than the user, so the cookie in the user's + // browser stays small and never has to change. + + // Also,--allow-pub-response is explained at + // https://docs.nats.io/running-a-nats-service/configuration/securing_nats/authorization#allow-responses-map + // and makes it so we don't have to allow any user to publish to + // all of _INBOX.>, which might be bad, since one user could in theory + // publish a response to a different user's request (though in practice + // the subject is random so not feasible). Defense in depth. + + const args = ["edit", "signing-key", "--sk", name, "--allow-pub-response"]; + let changed = false; if (rm.length > 0) { // --rm applies to both pub and sub and happens after adding, From f4cdd2a68d63f5a0f8ef4a39ccfab4ff4f699dc6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 12 Feb 2025 16:51:49 +0000 Subject: [PATCH 178/281] database user tracker: properly fix subtle bug that would cause issues for older projects when a user has more than 10000 projects --- .../postgres/project-and-user-tracker.ts | 53 ++++++++++++++----- src/packages/util/db-schema/projects.ts | 2 + 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/packages/database/postgres/project-and-user-tracker.ts b/src/packages/database/postgres/project-and-user-tracker.ts index ffabe0b831..e5b2f96fd6 100644 --- a/src/packages/database/postgres/project-and-user-tracker.ts +++ b/src/packages/database/postgres/project-and-user-tracker.ts @@ -14,14 +14,11 @@ */ import { EventEmitter } from "events"; - import { callback } from "awaiting"; import { callback2 } from "@cocalc/util/async-utils"; - import { close, len } from "@cocalc/util/misc"; - import { PostgreSQL, QueryOptions, QueryResult } from "./types"; - +import { getPoolClient } from "@cocalc/database/pool"; import { ChangeEvent, Changes } from "./changefeed"; const { all_results } = require("../postgres-base"); @@ -362,23 +359,53 @@ export class ProjectAndUserTracker extends EventEmitter { throw Error("do_register MUST NOT be called twice at once!"); this.do_register_lock = true; try { + // Register this account, which starts by getting ALL of their projects. + // 2021-05-10: it's possible that a single user has a really large number of projects, so + // we get the projects in batches to reduce the load on the database. + // We must have *all* projects, since this is used frequently in + // database/postgres-user-queries.coffee + // when deciding how to route listen/notify events to users. Search for + // "# Check that this is a project we have read access to" + // E.g., without all projects, changefeeds would just fail to update, + // which, e.g., makes it so projects appear to not start. // Register this account - let projects: QueryResult[]; + const client = await getPoolClient(); + let projects: QueryResult[] = []; + const batchSize = 2000; try { - // 2021-05-10: one user has a really large number of projects, which causes the hub to crash - // TODO: fix this ORDER BY .. LIMIT .. part properly - projects = await query(this.db, { - query: - "SELECT project_id, json_agg(o) as users FROM (SELECT project_id, jsonb_object_keys(users) AS o FROM projects WHERE users ? $1::TEXT ORDER BY last_edited DESC LIMIT 10000) s group by s.project_id", - params: [account_id], - }); + // Start a transaction + await client.query("BEGIN"); + // Declare a cursor + await client.query( + ` + DECLARE project_cursor CURSOR FOR SELECT project_id, json_agg(o) as users + FROM (SELECT project_id, jsonb_object_keys(users) AS o FROM projects + WHERE users ? $1::TEXT) AS s group by s.project_id`, + [account_id], + ); + // Fetch rows in batches + while (true) { + const batchResult = await client.query( + `FETCH ${batchSize} FROM project_cursor`, + ); + projects = projects.concat(batchResult.rows); + if (batchResult.rows.length < batchSize) { + break; // No more rows to fetch + } + } + // Close the cursor and end the transaction + await client.query("CLOSE project_cursor"); + await client.query("COMMIT"); } catch (err) { + // If an error occurs, roll back the transaction + await client.query("ROLLBACK"); const e = `error registering '${account_id}' -- err=${err}`; dbg(e); this.handle_error(e); // it is game over. return; + } finally { + client.release(); } - // we care about this account_id this.accounts[account_id] = true; diff --git a/src/packages/util/db-schema/projects.ts b/src/packages/util/db-schema/projects.ts index 35f861fed1..3512647835 100644 --- a/src/packages/util/db-schema/projects.ts +++ b/src/packages/util/db-schema/projects.ts @@ -349,8 +349,10 @@ if ( throw Error("make typescript happy"); } schema.projects_all.user_query.get.options = []; +schema.projects_all.user_query.get.options_load = []; schema.projects_all.virtual = "projects"; schema.projects_all.user_query.get.pg_where = ["projects"]; +schema.projects_all.user_query.get.pg_where_load = ["projects"]; // Table that provides extended read info about a single project // but *ONLY* for admin. From 29f1f5b7ca5e29fab577a49dac3720ca8bb49703 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 12 Feb 2025 18:04:50 +0000 Subject: [PATCH 179/281] use test namespace for testing nats kv and streams. --- src/packages/backend/jest.config.js | 7 ++++--- src/packages/backend/test/setup.js | 4 ++++ src/packages/nats/names.ts | 9 ++++++++- 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 src/packages/backend/test/setup.js diff --git a/src/packages/backend/jest.config.js b/src/packages/backend/jest.config.js index 140b9467f2..fdf02d2738 100644 --- a/src/packages/backend/jest.config.js +++ b/src/packages/backend/jest.config.js @@ -1,6 +1,7 @@ /** @type {import('ts-jest').JestConfigWithTsJest} */ module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - testMatch: ['**/?(*.)+(spec|test).ts?(x)'], + preset: "ts-jest", + testEnvironment: "node", + setupFiles: ["./test/setup.js"], + testMatch: ["**/?(*.)+(spec|test).ts?(x)"], }; diff --git a/src/packages/backend/test/setup.js b/src/packages/backend/test/setup.js new file mode 100644 index 0000000000..6ed5b82df9 --- /dev/null +++ b/src/packages/backend/test/setup.js @@ -0,0 +1,4 @@ +// test/setup.js + +// checked for in some code to behave differently while running unit tests. +process.env.COCALC_TEST_MODE = true; diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts index 153831ba7a..40816204a7 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/nats/names.ts @@ -36,7 +36,11 @@ export function jsName({ return `project-${project_id}`; } if (!account_id) { - return "public"; + if (process.env.COCALC_TEST_MODE) { + return "test"; + } else { + return "public"; + } } return `account-${account_id}`; } @@ -86,6 +90,9 @@ export function streamSubject({ return `project.${project_id}.stream.>`; } if (!account_id) { + if (process.env.COCALC_TEST_MODE) { + return "test.stream.>"; + } return "public.stream.>"; } return `account.${account_id}.stream.>`; From 26a7221686b437bd7de9c48a26a7ce6af518a2cd Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 12 Feb 2025 20:36:10 +0000 Subject: [PATCH 180/281] nats dstream: fix some bugs and create some unit tests --- .../backend/nats/test/dstream.test.ts | 171 ++++++++++++++++++ src/packages/nats/sync/dstream.ts | 20 +- src/packages/nats/sync/stream.ts | 3 + 3 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 src/packages/backend/nats/test/dstream.test.ts diff --git a/src/packages/backend/nats/test/dstream.test.ts b/src/packages/backend/nats/test/dstream.test.ts new file mode 100644 index 0000000000..8a28cdc530 --- /dev/null +++ b/src/packages/backend/nats/test/dstream.test.ts @@ -0,0 +1,171 @@ +/* +Testing basic ops with dsteam (distributed streams) + +DEVELOPMENT: + +pnpm exec jest --watch --forceExit --detectOpenHandles "dstream.test.ts" + +*/ + +import { dstream as createDstream } from "@cocalc/backend/nats/sync"; +import { once } from "@cocalc/util/async-utils"; +// import { delay } from "awaiting"; + +async function create() { + const name = `test-${Math.random()}`; + return await createDstream({ name, noAutosave: true }); +} + +describe("create a dstream and do some basic operations", () => { + let s; + + it("creates stream", async () => { + s = await create(); + }); + + it("starts out empty", () => { + expect(s.get()).toEqual([]); + expect(s.length).toEqual(0); + }); + + const mesg = { stdout: "hello" }; + it("publishes a message to the stream and confirms it is there", () => { + s.push(mesg); + expect(s.get()).toEqual([mesg]); + expect(s.length).toEqual(1); + expect(s[0]).toEqual(mesg); + }); + + it("confirm persistence: closes and re-opens stream and confirms message is still there", async () => { + const name = s.name; + await s.save(); + // close s: + await s.close(); + // using s fails + expect(s.get).toThrow("closed"); + // create new stream with same name + const t = await createDstream({ name }); + // ensure it is NOT just from the cache + expect(s === t).toBe(false); + // make sure it has our message + expect(t.get()).toEqual([mesg]); + }); +}); + +describe("create two dstreams and observe sync between them", () => { + const name = `test-${Math.random()}`; + let s1, s2; + it("creates two distinct dstream objects s1 and s2 with the same name", async () => { + s1 = await createDstream({ name, noAutosave: true }, { noCache: true }); + s2 = await createDstream({ name, noAutosave: true }, { noCache: true }); + // definitely distinct + expect(s1 === s2).toBe(false); + }); + + it("writes to s1 and observes s2 doesn't see anything until we save", async () => { + s1.push("hello"); + expect(s1[0]).toEqual("hello"); + expect(s2.length).toEqual(0); + s1.save(); + await once(s2, "change"); + expect(s2[0]).toEqual("hello"); + expect(s2.get()).toEqual(["hello"]); + }); + + it("now write to s2 and save and see that reflected in s1", async () => { + s2.push("hi from s2"); + s2.save(); + await once(s1, "change"); + expect(s1[1]).toEqual("hi from s2"); + }); + + it("write to s1 and s2 and save at the same time and see some 'random choice' of order gets imposed by the server", async () => { + s1.push("s1"); + s2.push("s2"); + // our changes are reflected locally + expect(s1.get()).toEqual(["hello", "hi from s2", "s1"]); + expect(s2.get()).toEqual(["hello", "hi from s2", "s2"]); + // now kick off the two saves *in parallel* + s1.save(); + s2.save(); + await once(s1, "change"); + if (s2.length != s1.length) { + await once(s2, "change"); + } + expect(s1.get()).toEqual(s2.get()); + // in fact s1,s2 is the order since we called s1.save first: + expect(s1.get()).toEqual(["hello", "hi from s2", "s1", "s2"]); + }); +}); + +describe("get sequence number and time of message", () => { + let s; + + it("creates stream and write message", async () => { + s = await create(); + s.push("hello"); + }); + + it("sequence number is initialized undefined because it is server assigned ", async () => { + const n = s.seq(0); + expect(n).toBe(undefined); + }); + + it("time also undefined because it is server assigned ", async () => { + const t = s.time(0); + expect(t).toBe(undefined); + }); + + it("save and get server assigned sequence number", async () => { + await s.save(); + const n = s.seq(0); + expect(n).toBeGreaterThan(0); + }); + + it("save and get server assigned time", async () => { + await s.save(); + const t = s.time(0); + // since testing on the same machine as server, these times should be close: + expect(t.getTime() - Date.now()).toBeLessThan(5000); + }); + + it("publish another message and get next server number is bigger", async () => { + const n = s.seq(0); + s.push("there"); + await s.save(); + const m = s.seq(1); + expect(m).toBeGreaterThan(n); + }); + + it("and time is bigger", async () => { + expect(s.time(0).getTime()).toBeLessThan(s.time(1).getTime()); + }); +}); + +describe("closing also saves by default, but not if autosave is off", () => { + let s; + const name = `test-${Math.random()}`; + + it("creates stream and write a message", async () => { + s = await createDstream({ name, noAutosave: false /* the default */ }); + s.push(389); + }); + + it("closes then opens and message is there, since autosave is on", async () => { + await s.close(); + const t = await createDstream({ name }); + expect(t[0]).toEqual(389); + }); + + it("make another stream with autosave off, and close which causes LOSS OF DATA", async () => { + const name = `test-${Math.random()}`; + const s = await createDstream({ name, noAutosave: true }); + s.push(389); + s.close(); + const t = await createDstream({ name, noAutosave: true }); + // data is gone forever! + expect(t.length).toBe(0); + }); +}); + + diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 85ffb94cab..3dae407823 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -34,15 +34,21 @@ import { millis } from "@cocalc/nats/util"; const MAX_PARALLEL = 50; +export interface DStreamOptions extends StreamOptions { + noAutosave?: boolean; +} + export class DStream extends EventEmitter { public readonly name: string; private stream?: Stream; private messages: any[]; private raw: any[]; + private noAutosave: boolean; private local: { [id: string]: { mesg: any; subject?: string } } = {}; - constructor(opts: StreamOptions) { + constructor(opts: DStreamOptions) { super(); + this.noAutosave = !!opts.noAutosave; this.name = opts.name; this.stream = new Stream(opts); this.messages = this.stream.messages; @@ -67,10 +73,13 @@ export class DStream extends EventEmitter { this.emit("connected"); }); - close = () => { + close = async () => { if (this.stream == null) { return; } + if (!this.noAutosave) { + await this.save(); + } this.stream.close(); this.emit("closed"); this.removeAllListeners(); @@ -84,6 +93,9 @@ export class DStream extends EventEmitter { }; get = (n?) => { + if (this.stream == null) { + throw Error("closed"); + } if (n == null) { return [ ...this.messages, @@ -117,7 +129,9 @@ export class DStream extends EventEmitter { publish = (mesg, subject?: string) => { const id = randomId(); this.local[id] = { mesg, subject }; - this.save(); + if (!this.noAutosave) { + this.save(); + } }; push = (...args) => { diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 05c351dc1a..6015079dcc 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -208,6 +208,9 @@ export class Stream extends EventEmitter { }); get = (n?) => { + if (this.js == null) { + throw Error("closed"); + } if (n == null) { return [...this.messages]; } else { From 4994b843725b615635a8f5397aa1186cbb5b4a8a Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 13 Feb 2025 00:09:34 +0000 Subject: [PATCH 181/281] nats dstream: add unit tests and fix bugs --- .../backend/nats/test/dstream.test.ts | 69 ++++++++++++++++++- src/packages/nats/sync/dstream.ts | 46 ++++++++++--- src/packages/nats/sync/stream.ts | 2 +- 3 files changed, 102 insertions(+), 15 deletions(-) diff --git a/src/packages/backend/nats/test/dstream.test.ts b/src/packages/backend/nats/test/dstream.test.ts index 8a28cdc530..2a642cd5f5 100644 --- a/src/packages/backend/nats/test/dstream.test.ts +++ b/src/packages/backend/nats/test/dstream.test.ts @@ -36,6 +36,14 @@ describe("create a dstream and do some basic operations", () => { expect(s[0]).toEqual(mesg); }); + it("verifies that unsaved changes works properly", async () => { + expect(s.hasUnsavedChanges()).toBe(true); + expect(s.unsavedChanges()).toEqual([mesg]); + await s.save(); + expect(s.hasUnsavedChanges()).toBe(false); + expect(s.unsavedChanges()).toEqual([]); + }); + it("confirm persistence: closes and re-opens stream and confirms message is still there", async () => { const name = s.name; await s.save(); @@ -117,13 +125,13 @@ describe("get sequence number and time of message", () => { }); it("save and get server assigned sequence number", async () => { - await s.save(); + s.save(); + await once(s, "change"); const n = s.seq(0); expect(n).toBeGreaterThan(0); }); - it("save and get server assigned time", async () => { - await s.save(); + it("get server assigned time", async () => { const t = s.time(0); // since testing on the same machine as server, these times should be close: expect(t.getTime() - Date.now()).toBeLessThan(5000); @@ -168,4 +176,59 @@ describe("closing also saves by default, but not if autosave is off", () => { }); }); +describe("testing start_seq", () => { + const name = `test-${Math.random()}`; + let seq; + it("creates a stream and adds 3 messages, noting their assigned sequence numbers", async () => { + const s = await createDstream({ name, noAutosave: true }); + s.push(1, 2, 3); + expect(s.get()).toEqual([1, 2, 3]); + // save, thus getting sequence numbers + s.save(); + while (s.seq(2) == null) { + s.save(); + await once(s, "change"); + } + seq = [s.seq(0), s.seq(1), s.seq(2)]; + // tests partly that these are integers... + const n = seq.reduce((a, b) => a + b, 0); + expect(typeof n).toBe("number"); + expect(n).toBeGreaterThan(2); + await s.close(); + }); + + let s; + it("it opens the stream but starting with the last sequence number, so only one message", async () => { + s = await createDstream({ + name, + noAutosave: true, + start_seq: seq[2], + }); + expect(s.length).toBe(1); + expect(s.get()).toEqual([3]); + }); + + it("it then pulls in the previous message, so now two messages are loaded", async () => { + await s.load({ start_seq: seq[1] }); + expect(s.length).toBe(2); + expect(s.get()).toEqual([2, 3]); + }); +}); +describe("a little bit of a stress test", () => { + const name = `test-${Math.random()}`; + const count = 250; + let s; + it(`creates a stream and pushes ${count} messages`, async () => { + s = await createDstream({ + name, + noAutosave: true, + }); + for (let i = 0; i < count; i++) { + s.push({ i }); + } + expect(s.length).toBe(count); + await s.save(); + expect(s.length).toBe(count); + }); +}); diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 3dae407823..6e0fb151d8 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -44,7 +44,9 @@ export class DStream extends EventEmitter { private messages: any[]; private raw: any[]; private noAutosave: boolean; + // TODO: using Map for these will be better because we use .length a bunch, which is O(n) instead of O(1). private local: { [id: string]: { mesg: any; subject?: string } } = {}; + private saved: { [seq: number]: any } = {}; constructor(opts: DStreamOptions) { super(); @@ -66,8 +68,9 @@ export class DStream extends EventEmitter { if (this.stream == null) { throw Error("closed"); } - this.stream.on("change", (...args) => { - this.emit("change", ...args); + this.stream.on("change", (mesg, raw) => { + delete this.saved[raw.seq]; + this.emit("change", mesg); }); await this.stream.init(); this.emit("connected"); @@ -99,19 +102,31 @@ export class DStream extends EventEmitter { if (n == null) { return [ ...this.messages, + ...Object.values(this.saved), ...Object.values(this.local).map((x) => x.mesg), ]; } else { - return ( - this.messages[n] ?? - Object.values(this.local)[n - this.messages.length]?.mesg - ); + if (n < this.messages.length) { + return this.messages[n]; + } + const v = Object.keys(this.saved); + if (n < v.length + this.messages.length) { + return v[n - this.messages.length]; + } + return Object.values(this.local)[n - this.messages.length - v.length] + ?.mesg; } }; // sequence number of n-th message seq = (n) => { - return this.raw[n]?.seq; + if (n < this.raw.length) { + return this.raw[n]?.seq; + } + const v = Object.keys(this.saved); + if (n < v.length + this.raw.length) { + return parseInt(v[n - this.raw.length]); + } }; time = (n) => { @@ -123,7 +138,11 @@ export class DStream extends EventEmitter { }; get length() { - return this.messages.length + Object.keys(this.local).length; + return ( + this.messages.length + + Object.keys(this.saved).length + + Object.keys(this.local).length + ); } publish = (mesg, subject?: string) => { @@ -151,7 +170,7 @@ export class DStream extends EventEmitter { }; unsavedChanges = () => { - return Object.values(this.local); + return Object.values(this.local).map(({ mesg }) => mesg); }; save = reuseInFlight(async () => { @@ -180,7 +199,11 @@ export class DStream extends EventEmitter { const { mesg, subject } = this.local[id]; try { // @ts-ignore - await this.stream.publish(mesg, subject, { msgID: id }); + const { seq } = await this.stream.publish(mesg, subject, { msgID: id }); + if ((this.raw[this.raw.length - 1]?.seq ?? -1) < seq) { + // it still isn't in this.raw + this.saved[seq] = mesg; + } delete this.local[id]; } catch (err) { if (err.code == "REJECT") { @@ -197,7 +220,8 @@ export class DStream extends EventEmitter { await awaitMap(Object.keys(this.local), MAX_PARALLEL, f); }); - load = async (opts) => { + // load older messages starting at start_seq + load = async (opts: { start_seq: number }) => { if (this.stream == null) { throw Error("closed"); } diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 6015079dcc..3e1f572856 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -396,7 +396,7 @@ export class Stream extends EventEmitter { this.messages.push(mesg); this.raw.push(raw); if (!noEmit) { - this.emit("change", mesg); + this.emit("change", mesg, raw); } }; From e4c10ec1fa52bc767ea395e0c045488c908b6a83 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 13 Feb 2025 00:23:10 +0000 Subject: [PATCH 182/281] nats: adjust parallel param and make comment in unit tests about speed --- src/packages/backend/nats/test/dstream.test.ts | 5 +++-- src/packages/nats/sync/dstream.ts | 10 ++++++++-- src/packages/nats/sync/general-dkv.ts | 2 +- src/packages/nats/sync/general-kv.ts | 2 +- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/packages/backend/nats/test/dstream.test.ts b/src/packages/backend/nats/test/dstream.test.ts index 2a642cd5f5..d2e0837abd 100644 --- a/src/packages/backend/nats/test/dstream.test.ts +++ b/src/packages/backend/nats/test/dstream.test.ts @@ -9,7 +9,6 @@ pnpm exec jest --watch --forceExit --detectOpenHandles "dstream.test.ts" import { dstream as createDstream } from "@cocalc/backend/nats/sync"; import { once } from "@cocalc/util/async-utils"; -// import { delay } from "awaiting"; async function create() { const name = `test-${Math.random()}`; @@ -217,7 +216,7 @@ describe("testing start_seq", () => { describe("a little bit of a stress test", () => { const name = `test-${Math.random()}`; - const count = 250; + const count = 100; let s; it(`creates a stream and pushes ${count} messages`, async () => { s = await createDstream({ @@ -228,6 +227,8 @@ describe("a little bit of a stress test", () => { s.push({ i }); } expect(s.length).toBe(count); + // NOTE: warning -- this is **MUCH SLOWER**, e.g., 10x slower, + // running under jest, hence why count is small. await s.save(); expect(s.length).toBe(count); }); diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 6e0fb151d8..799f33340b 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -32,7 +32,7 @@ import { isNumericString } from "@cocalc/util/misc"; import { sha1 } from "@cocalc/util/misc"; import { millis } from "@cocalc/nats/util"; -const MAX_PARALLEL = 50; +const MAX_PARALLEL = 250; export interface DStreamOptions extends StreamOptions { noAutosave?: boolean; @@ -217,7 +217,13 @@ export class DStream extends EventEmitter { }; // NOTE: ES6 spec guarantees "String keys are returned in the order // in which they were added to the object." - await awaitMap(Object.keys(this.local), MAX_PARALLEL, f); + const ids = Object.keys(this.local); + const t = Date.now(); + await awaitMap(ids, MAX_PARALLEL, f); + console.log( + `saving ${ids.length} messages ${MAX_PARALLEL} at once took `, + Date.now() - t, + ); }); // load older messages starting at start_seq diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index d449f1e5eb..2cf8d9eb99 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -79,7 +79,7 @@ import { delay } from "awaiting"; import { map as awaitMap } from "awaiting"; export const TOMBSTONE = Symbol("tombstone"); -const MAX_PARALLEL = 50; +const MAX_PARALLEL = 250; export type MergeFunction = (opts: { key: string; diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index 442df3c949..085eee82e9 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -111,7 +111,7 @@ class RejectError extends Error { key: string; } -const MAX_PARALLEL = 50; +const MAX_PARALLEL = 250; // Note that the limit options are named in exactly the same was as for streams, // which is convenient for consistency. This is not consistent with NATS's From 825a16cf4b0647d637db0ca45b180bcf7f09726c Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 13 Feb 2025 00:40:29 +0000 Subject: [PATCH 183/281] nats dkv: better testing null/undefined values (and fix bug testing reveals) --- src/packages/backend/nats/test/dkv.test.ts | 46 ++++++++++++++++++++++ src/packages/nats/sync/dkv.ts | 8 ++++ src/packages/nats/sync/dstream.ts | 5 --- src/packages/nats/sync/kv.ts | 2 +- 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/src/packages/backend/nats/test/dkv.test.ts b/src/packages/backend/nats/test/dkv.test.ts index 7337f1ecb1..98ed70fad4 100644 --- a/src/packages/backend/nats/test/dkv.test.ts +++ b/src/packages/backend/nats/test/dkv.test.ts @@ -2,6 +2,7 @@ Testing basic ops with dkv DEVELOPMENT: + pnpm exec jest --watch --forceExit --detectOpenHandles "dkv.test.ts" */ @@ -295,3 +296,48 @@ describe("create many distinct clients at once, write to all of them, and see th } }); }); + +describe("tests involving null/undefined values", () => { + let kv1; + let kv2; + const name = `test-${Math.random()}`; + + it("creates the dkv twice", async () => { + kv1 = await createDkv({ name, noAutosave: true }, { noCache: true }); + kv2 = await createDkv({ name, noAutosave: true }, { noCache: true }); + expect(kv1.get()).toEqual({}); + expect(kv1 === kv2).toBe(false); + }); + + it("sets a value to null, which is fully supported like any other value", () => { + kv1.a = null; + expect(kv1.a).toBe(null); + expect(kv1.a === null).toBe(true); + expect(kv1.length).toBe(1); + }); + + it("make sure null value sync's as expected", async () => { + kv1.save(); + await once(kv2, "change"); + expect(kv2.a).toBe(null); + expect(kv2.a === null).toBe(true); + expect(kv2.length).toBe(1); + }); + + it("sets a value to undefined, which is the same as deleting a value", () => { + kv1.a = undefined; + expect(kv1.a).toBe(undefined); + expect(kv1.a === undefined).toBe(true); + expect(kv1.length).toBe(0); + expect(kv1.get()).toEqual({}); + }); + + it("make sure undefined (i.e., delete) sync's as expected", async () => { + kv1.save(); + await once(kv2, "change"); + expect(kv2.a).toBe(undefined); + expect(kv2.a === undefined).toBe(true); + expect(kv2.length).toBe(0); + expect(kv1.get()).toEqual({}); + }); +}); diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index 1c86e4a78d..74992faa20 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -220,6 +220,14 @@ export class DKV extends EventEmitter { if (this.generalDKV == null) { throw Error("closed"); } + if (value === undefined) { + // undefined can't be JSON encoded, so we can't possibly represent it, and this + // *must* be treated as a delete. + // NOTE that jc.encode encodes null and undefined the same, so supporting this + // as a value is just begging for misery. + this.delete(key); + return; + } this.generalDKV.set(`${this.prefix}.${this.sha1(key)}`, { key, value }); }; diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 799f33340b..9ee739e81b 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -218,12 +218,7 @@ export class DStream extends EventEmitter { // NOTE: ES6 spec guarantees "String keys are returned in the order // in which they were added to the object." const ids = Object.keys(this.local); - const t = Date.now(); await awaitMap(ids, MAX_PARALLEL, f); - console.log( - `saving ${ids.length} messages ${MAX_PARALLEL} at once took `, - Date.now() - t, - ); }); // load older messages starting at start_seq diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index 5f50cd1f54..c9fcf2848c 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -1,7 +1,7 @@ /* Always Consistent Centralized Key Value Store -NOTE: I think this isn't used by anything actually. Note it doesn't emit +NOTE: I think this isn't used by anything actually. Note it doesn't emit change events. Maybe we should delete this. DEVELOPMENT: From f3c53f6df32dfa426fc5bc2a3daa0c341f94925d Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 13 Feb 2025 01:03:05 +0000 Subject: [PATCH 184/281] nats: fix the @cocalc/sync test suite by not making *it* use nats --- src/packages/sync/editor/generic/sync-doc.ts | 5 ++++- src/packages/sync/jest.config.js | 1 + src/packages/sync/table/synctable.ts | 6 +++--- src/packages/sync/test/setup.js | 4 ++++ 4 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 src/packages/sync/test/setup.js diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index a6cf79352b..e840fb835b 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -19,7 +19,7 @@ EVENTS: - ... TODO */ -const USE_NATS = true; +const USE_NATS = true && !process.env.COCALC_TEST_MODE; /* OFFLINE_THRESH_S - If the client becomes disconnected from the backend for more than this long then---on reconnect---do @@ -282,6 +282,9 @@ export class SyncDoc extends EventEmitter { this[field] = opts[field]; } } + // NOTE: Do not use nats in test mode, since there we use a minimal "fake" client + // that does all communication internally without a network. + this.useNats = USE_NATS; if (this.ephemeral) { // So the doctype written to the database reflects the diff --git a/src/packages/sync/jest.config.js b/src/packages/sync/jest.config.js index 6d25a84365..f8f66c15d0 100644 --- a/src/packages/sync/jest.config.js +++ b/src/packages/sync/jest.config.js @@ -13,4 +13,5 @@ module.exports = { testMatch: ["**/__tests__/**/*.[t]s?(x)", "**/?(*.)+(spec|test).[t]s?(x)"], testPathIgnorePatterns: ["/node_modules/"], moduleFileExtensions: ["ts", "tsx", "js", "jsx", "json", "node"], + setupFiles: ["./test/setup.js"], }; diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index 56c0763de2..9768948ac6 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -19,7 +19,7 @@ let DEBUG: boolean = false; // enable experimental nats database backed changefeed. // for this to work you must explicitly run the server in @cocalc/database/nats/changefeeds -const USE_NATS = true; +const USE_NATS = true && !process.env.COCALC_TEST_MODE; export function set_debug(x: boolean): void { DEBUG = x; @@ -730,7 +730,7 @@ export class SyncTable extends EventEmitter { delete this.changefeed; } - private async create_changefeed_connection(): Promise { + private create_changefeed_connection = async (): Promise => { let delay_ms: number = 500; while (true) { this.close_changefeed(); @@ -763,7 +763,7 @@ export class SyncTable extends EventEmitter { } } } - } + }; private async wait_until_ready_to_query_db(): Promise { const dbg = this.dbg("wait_until_ready_to_query_db"); diff --git a/src/packages/sync/test/setup.js b/src/packages/sync/test/setup.js new file mode 100644 index 0000000000..6ed5b82df9 --- /dev/null +++ b/src/packages/sync/test/setup.js @@ -0,0 +1,4 @@ +// test/setup.js + +// checked for in some code to behave differently while running unit tests. +process.env.COCALC_TEST_MODE = true; From 1e60bd48b516cffd0832d5bd818df6f5a91f5e0c Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 13 Feb 2025 01:24:34 +0000 Subject: [PATCH 185/281] fixing unit tests --- src/packages/backend/nats/test/dkv.test.ts | 4 +-- .../backend/nats/test/dstream.test.ts | 3 ++ src/packages/nats/package.json | 1 - .../control/stop-idle-projects.test.ts | 28 +++++++++---------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/packages/backend/nats/test/dkv.test.ts b/src/packages/backend/nats/test/dkv.test.ts index 98ed70fad4..1edc06ec83 100644 --- a/src/packages/backend/nats/test/dkv.test.ts +++ b/src/packages/backend/nats/test/dkv.test.ts @@ -101,9 +101,7 @@ describe("check server assigned times", () => { kv.a = { b: 7 }; // not serve assigned yet expect(kv.time("a")).toEqual(undefined); - await kv.save(); - // still not server assigned - expect(kv.time("a")).toEqual(undefined); + kv.save(); await once(kv, "change"); // now we must have it. // sanity check: within a second diff --git a/src/packages/backend/nats/test/dstream.test.ts b/src/packages/backend/nats/test/dstream.test.ts index d2e0837abd..34d2b4caf9 100644 --- a/src/packages/backend/nats/test/dstream.test.ts +++ b/src/packages/backend/nats/test/dstream.test.ts @@ -145,6 +145,9 @@ describe("get sequence number and time of message", () => { }); it("and time is bigger", async () => { + if (s.time(1) == null) { + await once(s, "change"); + } expect(s.time(0).getTime()).toBeLessThan(s.time(1).getTime()); }); }); diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 94108bdd88..d23e151915 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -14,7 +14,6 @@ "preinstall": "npx only-allow pnpm", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", - "test": "pnpm exec jest", "prepublishOnly": "pnpm test" }, "files": ["dist/**", "README.md", "package.json"], diff --git a/src/packages/server/projects/control/stop-idle-projects.test.ts b/src/packages/server/projects/control/stop-idle-projects.test.ts index 357aaf2731..a167ce06c2 100644 --- a/src/packages/server/projects/control/stop-idle-projects.test.ts +++ b/src/packages/server/projects/control/stop-idle-projects.test.ts @@ -20,11 +20,11 @@ afterAll(async () => { describe("creates a project, set various parameters, and runs idle project function, it and confirm that things work as intended", () => { let project_id; - const projectsTheGotStopped = new Set([]); + const projectsThatGotStopped = new Set([]); const stopProject = async (project_id) => { - projectsTheGotStopped.add(project_id); + projectsThatGotStopped.add(project_id); }; - const reset = () => projectsTheGotStopped.clear(); + const reset = () => projectsThatGotStopped.clear(); const pool = getPool(); afterAll(async () => { @@ -38,17 +38,17 @@ describe("creates a project, set various parameters, and runs idle project funct it("confirm that our project doesn't get stopped", async () => { await stopIdleProjects(stopProject); - expect(projectsTheGotStopped.has(project_id)).toBe(false); + expect(projectsThatGotStopped.has(project_id)).toBe(false); }); it("mock start of our project by setting run_quota, last_edited, and last_started", async () => { await pool.query( - `UPDATE projects SET run_quota='{"network": false, "cpu_limit": 1, "disk_quota": 3000, "privileged": false, "cpu_request": 0.02, "member_host": false, "dedicated_vm": false, "idle_timeout": 1800, "memory_limit": 1000, "always_running": false, "memory_request": 200, "dedicated_disks": []}', + `UPDATE projects SET run_quota='{"network": false, "cpu_limit": 1, "disk_quota": 3000, "privileged": false, "cpu_request": 0.02, "member_host": false, "dedicated_vm": false, "idle_timeout": 1800, "memory_limit": 1000, "always_running": false, "memory_request": 200, "dedicated_disks": []}', last_edited=NOW(), last_started=NOW(), state='{"state":"running"}' WHERE project_id=$1`, [project_id] ); await stopIdleProjects(stopProject); - expect(projectsTheGotStopped.has(project_id)).toBe(false); + expect(projectsThatGotStopped.has(project_id)).toBe(false); }); it("changes our project so that last_edited is an hour ago and last_started is an hour ago, and observe project gets stopped", async () => { @@ -57,7 +57,7 @@ describe("creates a project, set various parameters, and runs idle project funct [project_id] ); await stopIdleProjects(stopProject); - expect(projectsTheGotStopped.has(project_id)).toBe(true); + expect(projectsThatGotStopped.has(project_id)).toBe(true); }); it("changes our project so that last_edited is an hour ago and last_started is a minute ago, and observe project does NOT get stopped", async () => { @@ -67,7 +67,7 @@ describe("creates a project, set various parameters, and runs idle project funct ); reset(); await stopIdleProjects(stopProject); - expect(projectsTheGotStopped.has(project_id)).toBe(false); + expect(projectsThatGotStopped.has(project_id)).toBe(false); }); it("changes our project so that last_edited is a minute ago and last_started is an hour ago, and observe project does NOT get stopped", async () => { @@ -77,7 +77,7 @@ describe("creates a project, set various parameters, and runs idle project funct ); reset(); await stopIdleProjects(stopProject); - expect(projectsTheGotStopped.has(project_id)).toBe(false); + expect(projectsThatGotStopped.has(project_id)).toBe(false); }); it("changes our project so that last_edited and last_started are both a month ago, but always_running is true, and observe project does NOT get stopped", async () => { @@ -88,19 +88,19 @@ describe("creates a project, set various parameters, and runs idle project funct ); reset(); await stopIdleProjects(stopProject); - expect(projectsTheGotStopped.has(project_id)).toBe(false); + expect(projectsThatGotStopped.has(project_id)).toBe(false); }); it("makes it so stopping the project throws an error, and checks that the entire stopIdleProjects does NOT throw an error (it just logs something)", async () => { await pool.query( `UPDATE projects SET run_quota='{"network": false, "cpu_limit": 1, "disk_quota": 3000, "privileged": false, "cpu_request": 0.02, "member_host": false, "dedicated_vm": false, "idle_timeout": 1800, "memory_limit": 1000, "always_running": false, "memory_request": 200, "dedicated_disks": []}', - last_edited=NOW()-interval '1 month', last_started=NOW()-interval '1 month' WHERE project_id=$1`, + last_edited=NOW()-interval '1 month', last_started=NOW()-interval '1 month', state='{"state":"running"}' WHERE project_id=$1`, [project_id] ); - // first confirm it will get called + // first confirm stopProject2 will get called reset(); await stopIdleProjects(stopProject); - expect(projectsTheGotStopped.has(project_id)).toBe(true); + expect(projectsThatGotStopped.has(project_id)).toBe(true); // now call again with error but doesn't break anything const stopProject2 = async (project_id) => { await stopProject(project_id); @@ -108,6 +108,6 @@ describe("creates a project, set various parameters, and runs idle project funct }; reset(); await stopIdleProjects(stopProject2); - expect(projectsTheGotStopped.has(project_id)).toBe(true); + expect(projectsThatGotStopped.has(project_id)).toBe(true); }); }); From d426771855ebb5f442dc07d361f00776ec787aff Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 13 Feb 2025 01:25:27 +0000 Subject: [PATCH 186/281] delete the exec shell code api test - we have replaced the api with something nats based, which we could test differently, and this was just a proof of concept test to start things out for the old approach --- src/packages/project/exec_shell_code.test.ts | 32 -------------------- 1 file changed, 32 deletions(-) delete mode 100644 src/packages/project/exec_shell_code.test.ts diff --git a/src/packages/project/exec_shell_code.test.ts b/src/packages/project/exec_shell_code.test.ts deleted file mode 100644 index 3fc70fb1f3..0000000000 --- a/src/packages/project/exec_shell_code.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -// jest test if calling something in bash works - -import { exec_shell_code } from "@cocalc/project/exec_shell_code"; - -test("exec_shell_code", (done) => { - const mesg = { - id: "abf3b9ca-47e3-4d77-a0f7-eec04952c684", - event: "project_exec", - command: "echo $(( 2 * 21 ))", - bash: true, - }; - - // make socket a mock object with a method write_mesg - const socket = { - write_mesg: (type, mesg) => { - //console.log("type", type, "mesg", mesg); - expect(type).toBe("json"); - expect(mesg).toEqual({ - event: "project_exec_output", - id: mesg.id, - stdout: expect.any(String), - stderr: "", - exit_code: 0, - type: "blocking", - }); - expect(mesg.stdout).toBe("42\n"); - done(); - }, - }; - - exec_shell_code(socket as any, mesg); -}); From ae8957841473a6d3f5a7dd418c0437f0eec8e1b2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 13 Feb 2025 12:59:39 +0000 Subject: [PATCH 187/281] admin impersonation: fix some bugs in my new implementation --- .../crm-editor/tables/accounts.ts | 1 + .../crm-editor/tables/auth-tokens.ts | 21 ++++++++++ .../frame-editors/crm-editor/tables/tables.ts | 1 + src/packages/server/auth/auth-token.ts | 4 +- src/packages/util/db-schema/accounts.ts | 3 +- src/packages/util/db-schema/auth.ts | 39 +++++++++++++++++++ 6 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 src/packages/frontend/frame-editors/crm-editor/tables/auth-tokens.ts diff --git a/src/packages/frontend/frame-editors/crm-editor/tables/accounts.ts b/src/packages/frontend/frame-editors/crm-editor/tables/accounts.ts index e7e242cb03..c1949b13a6 100644 --- a/src/packages/frontend/frame-editors/crm-editor/tables/accounts.ts +++ b/src/packages/frontend/frame-editors/crm-editor/tables/accounts.ts @@ -30,6 +30,7 @@ register({ sign_up_usage_intent: null, balance_alert: null, auto_balance: null, + deleted: null, }, ], }, diff --git a/src/packages/frontend/frame-editors/crm-editor/tables/auth-tokens.ts b/src/packages/frontend/frame-editors/crm-editor/tables/auth-tokens.ts new file mode 100644 index 0000000000..c8cee2d377 --- /dev/null +++ b/src/packages/frontend/frame-editors/crm-editor/tables/auth-tokens.ts @@ -0,0 +1,21 @@ +import { register } from "./tables"; + +register({ + name: "auth_tokens", + + title: "Auth Tokens", + + icon: "key", + + query: { + crm_auth_tokens: [ + { + account_id: null, + expire: null, + created: null, + created_by: null, + is_admin: null, + }, + ], + }, +}); diff --git a/src/packages/frontend/frame-editors/crm-editor/tables/tables.ts b/src/packages/frontend/frame-editors/crm-editor/tables/tables.ts index 5442e7ace7..46e30868c4 100644 --- a/src/packages/frontend/frame-editors/crm-editor/tables/tables.ts +++ b/src/packages/frontend/frame-editors/crm-editor/tables/tables.ts @@ -18,6 +18,7 @@ import "./file-access-log"; import "./file-use"; import "./site-licenses"; import "./accounts"; +import "./auth-tokens"; import "./messages"; import "./agents"; import "./patches"; diff --git a/src/packages/server/auth/auth-token.ts b/src/packages/server/auth/auth-token.ts index ca00846b49..e76aac499a 100644 --- a/src/packages/server/auth/auth-token.ts +++ b/src/packages/server/auth/auth-token.ts @@ -42,8 +42,8 @@ export async function generateUserAuthToken({ const authToken = generate(24); const pool = getPool(); await pool.query( - "INSERT INTO auth_tokens (auth_token, expire, account_id) VALUES($1, NOW()+INTERVAL '12 hours', $2)", - [authToken, account_id], + "INSERT INTO auth_tokens (auth_token, expire, account_id, created_by, created, is_admin) VALUES($1, NOW()+INTERVAL '12 hours', $2, $3, NOW(), $4)", + [authToken, user_account_id, account_id, is_admin], ); await centralLog({ event: "auth-token", diff --git a/src/packages/util/db-schema/accounts.ts b/src/packages/util/db-schema/accounts.ts index 8324618c4c..bad8f652f3 100644 --- a/src/packages/util/db-schema/accounts.ts +++ b/src/packages/util/db-schema/accounts.ts @@ -704,6 +704,7 @@ Table({ salesloft_id: null, sign_up_usage_intent: null, owner_id: null, + deleted: null, }, }, set: { @@ -918,4 +919,4 @@ export interface UserSearchResult { email_address?: string; } -export const ACCOUNT_ID_COOKIE_NAME = 'account_id'; +export const ACCOUNT_ID_COOKIE_NAME = "account_id"; diff --git a/src/packages/util/db-schema/auth.ts b/src/packages/util/db-schema/auth.ts index f6332d3c8e..1ffd56ee05 100644 --- a/src/packages/util/db-schema/auth.ts +++ b/src/packages/util/db-schema/auth.ts @@ -4,6 +4,7 @@ */ import { Table } from "./types"; +import { SCHEMA as schema } from "./index"; Table({ name: "remember_me", @@ -37,13 +38,51 @@ Table({ pg_type: "CHAR(24)", }, account_id: { + desc: "User who this auth token grants access to become", type: "uuid", + render: { type: "account" }, }, expire: { type: "timestamp", + render: { type: "timestamp", editable: false }, }, + created: { + desc: "When this auth token was created", + type: "timestamp", + render: { type: "timestamp" }, + }, + created_by: { + desc: "User who created the auth token.", + type: "uuid", + render: { type: "account" }, + }, + is_admin: { + desc: "True if wser who created the auth token did so as an admin.", + type: "boolean", + }, + }, + rules: { + primary_key: "auth_token", }, +}); + +Table({ + name: "crm_auth_tokens", rules: { + virtual: "auth_tokens", primary_key: "auth_token", + user_query: { + get: { + admin: true, // only admins can do get queries on this table + fields: { + account_id: null, + expire: null, + created: null, + created_by: null, + is_admin: null, + }, + }, + }, }, + fields: schema.auth_tokens.fields, }); From c6961fcad70b48d0d4ae2794076d27fd5841c3ed Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 13 Feb 2025 14:27:21 +0000 Subject: [PATCH 188/281] nats open file data: improving it --- src/packages/backend/nats/sync.ts | 5 ++ src/packages/frontend/client/client.ts | 2 +- src/packages/nats/sync/open-files.ts | 60 ++++++++++---- src/packages/project/nats/index.ts | 3 +- src/packages/project/nats/open-files.ts | 101 +++++++++++++++--------- src/packages/project/nats/sync.ts | 11 ++- 6 files changed, 125 insertions(+), 57 deletions(-) diff --git a/src/packages/backend/nats/sync.ts b/src/packages/backend/nats/sync.ts index 41bf4485de..7535fdcd17 100644 --- a/src/packages/backend/nats/sync.ts +++ b/src/packages/backend/nats/sync.ts @@ -7,6 +7,7 @@ import { kv as createKV, type KV } from "@cocalc/nats/sync/kv"; import { dkv as createDKV, type DKV } from "@cocalc/nats/sync/dkv"; import { dko as createDKO, type DKO } from "@cocalc/nats/sync/dko"; import { getEnv } from "@cocalc/backend/nats/env"; +import { createOpenFiles, type OpenFiles } from "@cocalc/nats/sync/open-files"; export type { Stream, DStream, KV, DKV, DKO }; @@ -29,3 +30,7 @@ export async function dkv(opts, options?): Promise { export async function dko(opts): Promise { return await createDKO({ env: await getEnv(), ...opts }); } + +export async function openFiles(project_id: string): Promise { + return await createOpenFiles({ env: await getEnv(), project_id }); +} diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 49deb4f411..533c6e1812 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -355,7 +355,7 @@ class Client extends EventEmitter implements WebappClient { id?: number; }) => { const x = await this.nats_client.openFiles(project_id); - await x.touch({ path }); + await x.touch(path); }; } diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index beed0cac0c..acbaf0ac53 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -5,11 +5,12 @@ DEVELOPMENT: Change to packages/backend, since packages/nats doesn't have a way to connect: -~/cocalc/src/packages/backend node -> z = await require('@cocalc/nats/sync/open-files').createOpenFiles({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf', env:await require('@cocalc/backend/nats').getEnv()}) +~/cocalc/src/packages/backend$ node + +> z = await require('@cocalc/backend/nats/sync').openFiles('00847397-d6a8-4cb0-96a8-6ef64ac3e6cf') > z.touch({path:'a.txt'}) > z.get({path:'a.txt'}) -{ open: true, count: 1 } +{ open: true, count: 1, time:2025-02-09T16:37:20.713Z } > z.touch({path:'a.txt'}) > z.get({path:'a.txt'}) { open: true, count: 2 } @@ -20,6 +21,11 @@ Change to packages/backend, since packages/nats doesn't have a way to connect: { 'a.txt': { open: true, count: 3 }, 'foo/b.md': { open: true, count: 1 } + +Frontend Dev in browser: + +z = await cc.client.nats_client.openFiles('00847397-d6a8-4cb0-96a8-6ef64ac3e6cf') +z.get() } */ @@ -40,9 +46,7 @@ export interface Entry { // last time this entry was changed -- this is automatically set // correctly by the NATS server in a consistent way: // https://github.com/nats-io/nats-server/discussions/3095 - // It gets updated even if you set an object to itself (making no change). time?: Date; - // count?: number; } @@ -79,10 +83,12 @@ export class OpenFiles extends EventEmitter { }, }); this.dkv = d; - d.on("change", ({ key: path, value }) => { - const time = d.time(path); - const { open } = value ?? {}; - this.emit("change", { path, open, time } as Entry); + d.on("change", ({ key: path }) => { + const entry = this.get(path); + if (entry != null) { + // not deleted and timestamp is set: + this.emit("change", entry as Entry); + } }); this.setState("connected"); }; @@ -112,8 +118,7 @@ export class OpenFiles extends EventEmitter { // When a client has a file open, they should periodically // touch it to indicate that it is open. // updates timestamp and ensures open=true. - // do we need compute server? - touch = ({ path }: { path: string }) => { + touch = (path: string) => { const dkv = this.getDkv(); // n = sequence number to make sure a write happens, which updates // server assigned timestamp. @@ -121,20 +126,41 @@ export class OpenFiles extends EventEmitter { dkv.set(path, { open: true, count: count + 1 }); }; - closeFile = ({ path }: { path: string }) => { + setError = (path: string, err?: any) => { + const dkv = this.getDkv(); + if (!err) { + const current = { ...dkv.get(path) }; + delete current.error; + dkv.set(path, current); + } else { + const current = { ...dkv.get(path) }; + current.error = { time: Date.now(), error: `${err}` }; + dkv.set(path, current); + } + }; + + closeFile = (path: string) => { const dkv = this.getDkv(); dkv.set(path, { ...dkv.get(path), open: false }); }; - get = (obj?: { path: string }) => { - return this.getDkv().get(obj?.path); + getAll = (): Entry[] => { + const x = this.getDkv().get(); + return Object.keys(x).map((path) => { + return { ...x[path], path, time: this.time(path) }; + }); + }; + + get = (path: string): Entry => { + const x = this.getDkv().get(path); + return { ...x, path, time: this.time(path) }; }; - delete = ({ path }: { path: string }) => { + delete = (path) => { this.getDkv().delete(path); }; - time = (obj?: { path: string }) => { - return this.getDkv().time(obj?.path); + time = (path?: string) => { + return this.getDkv().time(path); }; } diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts index ab5bda0813..d194215bdb 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/nats/index.ts @@ -2,7 +2,8 @@ Start the NATS servers: - the new api -- legacy api +- the open files tracker +- websocket api (temporary/legacy shim) */ import { getLogger } from "@cocalc/project/logger"; diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 4598f658d9..cc3176f8fd 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -3,25 +3,30 @@ Handle opening files in a project to save/load from disk and also enable compute DEVELOPMENT: -0. From the browser, terminate this api server running in the project already, if any +0. From the browser, terminate open-files api service running in the project already, if any await cc.client.nats_client.projectApi({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}).system.terminate({service:'open-files'}) + // {status: 'terminated', service: 'open-files'} -Set env variables as in a project (see api/index.ts ), then: +Set env variables as in a project (see api/index.ts ), then in nodejs: -> require("@cocalc/project/nats/open-files").init() +> x = require("@cocalc/project/nats/open-files").init() +> x.openFiles.getAll(); +> Object.keys(x.openSyncDocs) +> s = x.openSyncDocs['z4.tasks'] +// now you can directly work with the syncdoc for a given file, +// but from the perspective of the project, not the browser! */ import { - createOpenFiles, - OpenFiles, - Entry, -} from "@cocalc/nats/sync/open-files"; + openFiles as createOpenFiles, + type OpenFiles, + type OpenFileEntry, +} from "@cocalc/project/nats/sync"; import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; -import { compute_server_id, project_id } from "@cocalc/project/data"; -import { getEnv } from "./env"; +import { /*compute_server_id,*/ project_id } from "@cocalc/project/data"; import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; import { getClient } from "@cocalc/project/client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; @@ -37,24 +42,32 @@ let openFiles: OpenFiles | null = null; export async function init() { logger.debug("init"); - openFiles = await createOpenFiles({ - project_id, - env: await getEnv(), - }); - const entries: { [path: string]: Entry } = {}; - closeIgnoredFiles(entries, openFiles); + openFiles = await createOpenFiles(); + + // initialize + for (const entry of openFiles.getAll()) { + handleChange(entry); + } + + // start loop to watch for and close files that aren't touched frequently: + closeIgnoredFilesLoop(); + + // handle changes openFiles.on("change", (entry) => { - entries[entry.path] = entry; handleChange(entry); }); + + // usefule for development + return { openFiles, openSyncDocs }; } export function terminate() { logger.debug("terminating open-files service"); - openFiles?.close(); for (const path in openSyncDocs) { closeSyncDoc(path); } + openFiles?.close(); + openFiles = null; } const openSyncDocs: { [path: string]: SyncDoc } = {}; @@ -65,26 +78,31 @@ function getCutoff() { return new Date(Date.now() - 2.5 * NATS_OPEN_FILE_TOUCH_INTERVAL); } -async function handleChange({ path, open, time }: Entry) { +async function handleChange({ path, open, time }: OpenFileEntry) { + logger.debug("handleChange", { path, open, time }); const syncDoc = openSyncDocs[path]; const isOpenHere = syncDoc != null; - const id = 0; // todo - if (id != compute_server_id) { - if (isOpenHere) { - // close it here - closeSyncDoc(path); - } - // no further responsibility - return; - } + // TODO: need another table with compute server mappings + // const id = 0; // todo + // if (id != compute_server_id) { + // if (isOpenHere) { + // // close it here + // logger.debug("handleChange: closing", { path }); + // closeSyncDoc(path); + // } + // // no further responsibility + // return; + // } if (!open) { if (isOpenHere) { + logger.debug("handleChange: closing", { path }); closeSyncDoc(path); } return; } if (time != null && open && time >= getCutoff()) { if (!isOpenHere) { + logger.debug("handleChange: opening", { path }); // users actively care about this file being opened HERE, but it isn't openSyncDoc(path); } @@ -92,7 +110,7 @@ async function handleChange({ path, open, time }: Entry) { } } -function supportAutoclose(path: string) { +function supportAutoclose(path: string): boolean { // this feels way too "hard coded"; alternatively, maybe we make the kernel or whatever // actually update the interest? or something else... if (path.endsWith(".ipynb.sage-jupyter2") || path.endsWith(".sagews")) { @@ -101,20 +119,29 @@ function supportAutoclose(path: string) { return true; } -async function closeIgnoredFiles(entries, openFiles) { - while (openFiles.state == "connected") { +async function closeIgnoredFilesLoop() { + while (openFiles != null && openFiles.state == "connected") { await delay(NATS_OPEN_FILE_TOUCH_INTERVAL); if (openFiles.state != "connected") { return; } - logger.debug("closeIgnoredFiles: checking..."); + const paths = Object.keys(openSyncDocs); + if (paths.length == 0) { + logger.debug("closeIgnoredFiles: no paths currently open"); + continue; + } + logger.debug( + "closeIgnoredFiles: checking", + paths.length, + "currently open paths...", + ); const cutoff = getCutoff(); - for (const path in entries) { - const entry = entries[path]; + for (const path of paths) { + const entry = openFiles.get(path); if ( + entry.time != null && entry.time <= cutoff && - !supportAutoclose(path) && - openSyncDocs[path] != null + supportAutoclose(path) ) { logger.debug("closeIgnoredFiles: closing due to inactivity", { path }); closeSyncDoc(path); @@ -133,8 +160,8 @@ const closeSyncDoc = reuseInFlight(async (path: string) => { try { await syncDoc.close(); } catch (err) { - // TODO: maybe this could get saved in a nats key-value store? logger.debug(`WARNING -- issue closing syncdoc -- ${err}`); + openFiles?.setError(path, err); } }); @@ -188,7 +215,7 @@ const openSyncDoc = reuseInFlight(async (path: string) => { async function getTypeAndOpts( syncstrings, ): Promise<{ type: string; opts: any }> { - global.z = {syncstrings} + // global.z = { syncstrings }; let s = syncstrings.get_one(); if (s == null) { await syncstrings.wait(() => { diff --git a/src/packages/project/nats/sync.ts b/src/packages/project/nats/sync.ts index e47413bcf2..9474ce2e11 100644 --- a/src/packages/project/nats/sync.ts +++ b/src/packages/project/nats/sync.ts @@ -8,8 +8,13 @@ import { dkv as createDKV, type DKV } from "@cocalc/nats/sync/dkv"; import { dko as createDKO, type DKO } from "@cocalc/nats/sync/dko"; import { getEnv } from "./env"; import { project_id } from "@cocalc/project/data"; +import { + createOpenFiles, + type OpenFiles, + Entry as OpenFileEntry, +} from "@cocalc/nats/sync/open-files"; -export type { Stream, DStream, KV, DKV }; +export type { Stream, DStream, KV, DKV, OpenFiles, OpenFileEntry }; export async function stream(opts): Promise { return await createStream({ project_id, env: await getEnv(), ...opts }); @@ -30,3 +35,7 @@ export async function dkv(opts): Promise { export async function dko(opts): Promise { return await createDKO({ project_id, env: await getEnv(), ...opts }); } + +export async function openFiles(): Promise { + return await createOpenFiles({ env: await getEnv(), project_id }); +} From 9ee9323116bf3a81e4723f0055eaf6808d36a203 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 13 Feb 2025 15:03:46 +0000 Subject: [PATCH 189/281] nats sync: fix opening non-string first time --- src/packages/project/nats/open-files.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index cc3176f8fd..67361e2288 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -11,9 +11,15 @@ DEVELOPMENT: Set env variables as in a project (see api/index.ts ), then in nodejs: -> x = require("@cocalc/project/nats/open-files").init() +DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:nats:open-files node + +> x = await require("@cocalc/project/nats/open-files").init(); Object.keys(x) +[ 'openFiles', 'openSyncDocs' ] + > x.openFiles.getAll(); + > Object.keys(x.openSyncDocs) + > s = x.openSyncDocs['z4.tasks'] // now you can directly work with the syncdoc for a given file, // but from the perspective of the project, not the browser! @@ -217,10 +223,11 @@ async function getTypeAndOpts( ): Promise<{ type: string; opts: any }> { // global.z = { syncstrings }; let s = syncstrings.get_one(); - if (s == null) { + if (s?.doctype == null) { + // wait until there is a syncstring and its doctype is set: await syncstrings.wait(() => { s = syncstrings.get_one(); - return s != null; + return s?.doctype != null; }); } const opts: any = {}; From 139bb21f640939e1170b1282432d1fe913e08d3c Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 13 Feb 2025 15:57:22 +0000 Subject: [PATCH 190/281] nats: unit test open-files --- src/packages/backend/nats/sync.ts | 4 +- .../backend/nats/test/open-files.test.ts | 138 ++++++++++++++++++ src/packages/nats/sync/open-files.ts | 57 ++++++-- 3 files changed, 186 insertions(+), 13 deletions(-) create mode 100644 src/packages/backend/nats/test/open-files.test.ts diff --git a/src/packages/backend/nats/sync.ts b/src/packages/backend/nats/sync.ts index 7535fdcd17..0485fb64e8 100644 --- a/src/packages/backend/nats/sync.ts +++ b/src/packages/backend/nats/sync.ts @@ -31,6 +31,6 @@ export async function dko(opts): Promise { return await createDKO({ env: await getEnv(), ...opts }); } -export async function openFiles(project_id: string): Promise { - return await createOpenFiles({ env: await getEnv(), project_id }); +export async function openFiles(project_id: string, opts?): Promise { + return await createOpenFiles({ env: await getEnv(), project_id, ...opts }); } diff --git a/src/packages/backend/nats/test/open-files.test.ts b/src/packages/backend/nats/test/open-files.test.ts new file mode 100644 index 0000000000..8235bc944f --- /dev/null +++ b/src/packages/backend/nats/test/open-files.test.ts @@ -0,0 +1,138 @@ +/* +Unit test basic functionality of the openFiles distributed key:value +store. Projects and compute servers use this to know what files +to open so they can fulfill their backend responsibilities: + - computation + - save to disk + - load from disk when file changes + +DEVELOPMENT: + +pnpm exec jest --watch --forceExit --detectOpenHandles "open-files.test.ts" + +*/ + +import { openFiles as createOpenFiles } from "@cocalc/backend/nats/sync"; +import { once } from "@cocalc/util/async-utils"; +import { delay } from "awaiting"; + +const project_id = "00000000-0000-4000-8000-000000000000"; +async function create() { + return await createOpenFiles(project_id, { noAutosave: true, noCache: true }); +} + +describe("create open file tracker and do some basic operations", () => { + let o1, o2; + let file1 = `${Math.random()}.txt`; + let file2 = `${Math.random()}.txt`; + + it("creates two open files tracker (tracking same project) and clear them", async () => { + o1 = await create(); + o2 = await create(); + // ensure caching disable so our sync tests are real + expect(o1.getDkv() === o2.getDkv()).toBe(false); + o1.clear(); + await o1.save(); + expect(o1.hasUnsavedChanges()).toBe(false); + o2.clear(); + while (o2.hasUnsavedChanges()) { + try { + // expected due to merge conflict and autosave being disabled. + await o2.save(); + } catch { + await delay(10); + } + } + }); + + it("confirm they are cleared", async () => { + expect(o1.getAll()).toEqual([]); + expect(o2.getAll()).toEqual([]); + }); + + it("touch file in one and observe change and timestamp getting assigned by server", async () => { + o1.touch(file1); + expect(o1.get(file1)?.time).toBe(undefined); + o1.save(); + if (o1.get(file1)?.time == null) { + await once(o1, "change", 250); + expect(o1.get(file1).path).toBe(file1); + } + }); + + it("touches file in one and observes change by OTHER", async () => { + o1.touch(file2); + expect(o1.get(file2)?.path).toBe(file2); + expect(o2.get(file2)).toBe(undefined); + o1.save(); + if (o2.get(file2) == null) { + await once(o2, "change", 250); + expect(o2.get(file2).path).toBe(file2); + expect(o2.get(file2).time == null).toBe(false); + } + }); + + it("get all in o2 sees both file1 and file2", async () => { + const v = o2.getAll(); + expect(v[0].path).toBe(file1); + expect(v[1].path).toBe(file2); + expect(v.length).toBe(2); + }); + + it("delete file1", async () => { + o1.delete(file1); + expect(o1.get(file1)).toBe(undefined); + expect(o1.getAll().length).toBe(1); + o1.save(); + await delay(1000); + if (o2.get(file1) != null) { + await once(o2, "change", 250); + } + expect(o2.get(file1)).toBe(undefined); + // should be 1 due to file2 still being there: + expect(o2.getAll().length).toBe(1); + }); + + it("closes file2", async () => { + expect(o2.get(file2).open).toBe(true); + o2.closeFile(file2); + expect(o2.get(file2).open).toBe(false); + o2.save(); + if (o1.get(file2).open) { + await once(o1, "change", 250); + } + expect(o1.get(file2).open).toBe(false); + }); + + it("touching a closed file re-opens it", async () => { + o2.touch(file2); + expect(o2.get(file2).open).toBe(true); + o2.save(); + if (!o1.get(file2).open) { + await once(o1, "change", 250); + } + expect(o1.get(file2).open).toBe(true); + }); + + it("sets an error", async () => { + o2.setError(file2, Error("test error")); + expect(o2.get(file2).error.error).toBe("Error: test error"); + expect(typeof o2.get(file2).error.time == "number").toBe(true); + expect(Math.abs(Date.now() - o2.get(file2).error.time)).toBeLessThan(10000); + o2.save(); + if (!o1.get(file2).error) { + await once(o1, "change", 250); + } + expect(o1.get(file2).error.error).toBe("Error: test error"); + }); + + it("clears the error", async () => { + o1.setError(file2); + expect(o1.get(file2).error).toBe(undefined); + o1.save(); + if (o2.get(file2).error) { + await once(o2, "change", 250); + } + expect(o2.get(file2).error).toBe(undefined); + }); +}); diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index acbaf0ac53..f6fff669a5 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -50,8 +50,15 @@ export interface Entry { count?: number; } -export async function createOpenFiles({ env, project_id }) { - const openFiles = new OpenFiles({ env, project_id }); +interface Options { + env: NatsEnv; + project_id: string; + noAutosave?: boolean; + noCache?: boolean; +} + +export async function createOpenFiles(opts: Options) { + const openFiles = new OpenFiles(opts); await openFiles.init(); return openFiles; } @@ -59,13 +66,17 @@ export async function createOpenFiles({ env, project_id }) { export class OpenFiles extends EventEmitter { private project_id: string; private env: NatsEnv; + private noCache?: boolean; + private noAutosave?: boolean; private dkv?: DKV; public state: "disconnected" | "connected" | "closed" = "disconnected"; - constructor({ env, project_id }: { env: NatsEnv; project_id: string }) { + constructor({ env, project_id, noAutosave, noCache }: Options) { super(); this.env = env; this.project_id = project_id; + this.noAutosave = noAutosave; + this.noCache = noCache; } private setState = (state: State) => { @@ -74,14 +85,18 @@ export class OpenFiles extends EventEmitter { }; init = async () => { - const d = await dkv({ - name: "open-files", - project_id: this.project_id, - env: this.env, - limits: { - max_age: nanos(MAX_AGE_MS), + const d = await dkv( + { + name: "open-files", + project_id: this.project_id, + env: this.env, + limits: { + max_age: nanos(MAX_AGE_MS), + }, + noAutosave: this.noAutosave, }, - }); + { noCache: this.noCache }, + ); this.dkv = d; d.on("change", ({ key: path }) => { const entry = this.get(path); @@ -139,6 +154,11 @@ export class OpenFiles extends EventEmitter { } }; + // causes file to be immediately closed on backend + // no matter what, unrelated to how many users have it + // open or what type of file it is. Obviously, frontend + // clients also may need to pay attention to this, since they + // can just immediately reopen the file. closeFile = (path: string) => { const dkv = this.getDkv(); dkv.set(path, { ...dkv.get(path), open: false }); @@ -151,8 +171,11 @@ export class OpenFiles extends EventEmitter { }); }; - get = (path: string): Entry => { + get = (path: string): Entry | undefined => { const x = this.getDkv().get(path); + if (x == null) { + return x; + } return { ...x, path, time: this.time(path) }; }; @@ -160,6 +183,18 @@ export class OpenFiles extends EventEmitter { this.getDkv().delete(path); }; + clear = () => { + this.getDkv().clear(); + }; + + save = async () => { + await this.getDkv().save(); + }; + + hasUnsavedChanges = () => { + return this.getDkv().hasUnsavedChanges(); + }; + time = (path?: string) => { return this.getDkv().time(path); }; From a6ae9c4b9770839e3a23adf514a647a1cb57b3ca Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 13 Feb 2025 23:03:14 +0000 Subject: [PATCH 191/281] nats sync objects: mostly improving caching and testing --- src/packages/backend/nats/sync.ts | 8 +- src/packages/backend/nats/test/dko.test.ts | 107 +++++++++++++++++ .../backend/nats/test/dkv-merge.test.ts | 8 +- src/packages/backend/nats/test/dkv.test.ts | 32 +++-- .../backend/nats/test/dstream.test.ts | 4 +- src/packages/frontend/client/client.ts | 5 +- src/packages/frontend/nats/client.ts | 86 ++++++-------- src/packages/nats/sync/dko.ts | 29 ++--- src/packages/nats/sync/dkv.ts | 35 ++---- src/packages/nats/sync/dstream.ts | 52 +++----- src/packages/nats/sync/general-kv.ts | 2 +- src/packages/nats/sync/kv.ts | 27 ++--- src/packages/nats/sync/open-files.ts | 20 ++-- src/packages/nats/sync/stream.ts | 40 +++---- src/packages/nats/sync/syncdoc-info.ts | 48 ++++++++ src/packages/nats/sync/synctable.ts | 59 +++++----- src/packages/nats/types.ts | 1 - src/packages/pnpm-lock.yaml | 3 + src/packages/project/client.ts | 3 +- src/packages/project/nats/open-files.ts | 81 +++---------- src/packages/project/nats/synctable.ts | 54 ++++----- src/packages/sync/client/sync-client.ts | 61 ++++------ src/packages/sync/package.json | 13 +- src/packages/sync/tsconfig.json | 2 +- .../util/db-schema/syncstring-schema.ts | 2 +- src/packages/util/refcache.ts | 111 ++++++++++++++++++ 26 files changed, 512 insertions(+), 381 deletions(-) create mode 100644 src/packages/backend/nats/test/dko.test.ts create mode 100644 src/packages/nats/sync/syncdoc-info.ts create mode 100644 src/packages/util/refcache.ts diff --git a/src/packages/backend/nats/sync.ts b/src/packages/backend/nats/sync.ts index 0485fb64e8..b698ea0a74 100644 --- a/src/packages/backend/nats/sync.ts +++ b/src/packages/backend/nats/sync.ts @@ -15,16 +15,16 @@ export async function stream(opts): Promise { return await createStream({ env: await getEnv(), ...opts }); } -export async function dstream(opts, options?): Promise { - return await createDstream({ env: await getEnv(), ...opts }, options); +export async function dstream(opts): Promise { + return await createDstream({ env: await getEnv(), ...opts }); } export async function kv(opts): Promise { return await createKV({ env: await getEnv(), ...opts }); } -export async function dkv(opts, options?): Promise { - return await createDKV({ env: await getEnv(), ...opts }, options); +export async function dkv(opts): Promise { + return await createDKV({ env: await getEnv(), ...opts }); } export async function dko(opts): Promise { diff --git a/src/packages/backend/nats/test/dko.test.ts b/src/packages/backend/nats/test/dko.test.ts new file mode 100644 index 0000000000..daa3e8c511 --- /dev/null +++ b/src/packages/backend/nats/test/dko.test.ts @@ -0,0 +1,107 @@ +/* +Testing basic ops with kv + +DEVELOPMENT: + +pnpm exec jest --watch --forceExit --detectOpenHandles "dko.test.ts" + +*/ + +import { dko as createDko } from "@cocalc/backend/nats/sync"; +import { once } from "@cocalc/util/async-utils"; + +describe("create a public kv and do basic operations", () => { + let kv; + const name = `test-${Math.random()}`; + + it("creates the kv", async () => { + kv = await createDko({ name }); + expect(kv.get()).toEqual({}); + }); + + it("adds a key to the kv", () => { + kv.a = { x: 10 }; + expect(kv.a).toEqual({ x: 10 }); + }); + + it("complains if value is not an object", () => { + expect(() => { + kv.x = 5; + }).toThrow("object"); + }); + + it("waits for the kv to be longterm saved, then closing and recreates the kv and verifies that the key is there.", async () => { + await kv.save(); + kv.close(); + kv = await createDko({ name }); + expect(kv.a).toEqual({ x: 10 }); + }); + + it("closes the kv", async () => { + kv.close(); + expect(kv.get).toThrow("closed"); + }); +}); + +describe("opens a kv twice and verifies the cached works and is reference counted", () => { + let kv1; + let kv2; + const name = `test-${Math.random()}`; + + it("creates the same kv twice", async () => { + kv1 = await createDko({ name }); + kv2 = await createDko({ name }); + expect(kv1.get()).toEqual({}); + expect(kv1 === kv2).toBe(true); + }); + + it("closes kv1 (one reference)", async () => { + kv1.close(); + expect(kv2.get).not.toThrow(); + }); + + it("closes kv2 (another reference)", async () => { + kv2.close(); + // really closed! + expect(kv2.get).toThrow("closed"); + }); + + it("create and see it is new now", async () => { + kv1 = await createDko({ name }); + expect(kv1 === kv2).toBe(false); + }); +}); + +describe("opens a kv twice at once and observe sync", () => { + let kv1; + let kv2; + const name = `test-${Math.random()}`; + + it("creates the kv twice", async () => { + kv1 = await createDko({ name, noCache: true }); + kv2 = await createDko({ name, noCache: true }); + expect(kv1.get()).toEqual({}); + expect(kv2.get()).toEqual({}); + expect(kv1 === kv2).toBe(false); + }); + + it("sets a value in one and sees that it is NOT instantly set in the other", () => { + kv1.a = { x: 25 }; + expect(kv2.a).toBe(undefined); + }); + + it("awaits save and then sees the value *eventually* appears in the other", async () => { + kv1.save(); + // initially not there. + while (kv2.a?.x === undefined) { + await once(kv2, "change"); + } + expect(kv2.a).toEqual(kv1.a); + }); + + it("close up", () => { + kv1.close(); + kv2.close(); + }); +}); + diff --git a/src/packages/backend/nats/test/dkv-merge.test.ts b/src/packages/backend/nats/test/dkv-merge.test.ts index d99c86ef6a..25f7b37b46 100644 --- a/src/packages/backend/nats/test/dkv-merge.test.ts +++ b/src/packages/backend/nats/test/dkv-merge.test.ts @@ -2,7 +2,9 @@ Testing merge conflicts with dkv DEVELOPMENT: + pnpm exec jest --watch --forceExit --detectOpenHandles "dkv-merge.test.ts" + */ import { dkv as createDkv } from "@cocalc/backend/nats/sync"; @@ -14,12 +16,10 @@ async function getKvs(opts?) { // We disable autosave so that we have more precise control of how conflicts // get resolved, etc. for testing purposes. const kv1 = await createDkv( - { name, noAutosave: true, ...opts }, - { noCache: true }, + { name, noAutosave: true, ...opts, noCache: true }, ); const kv2 = await createDkv( - { name, noAutosave: true, ...opts }, - { noCache: true }, + { name, noAutosave: true, ...opts, noCache: true }, ); return { kv1, kv2 }; } diff --git a/src/packages/backend/nats/test/dkv.test.ts b/src/packages/backend/nats/test/dkv.test.ts index 1edc06ec83..2c2f9d868b 100644 --- a/src/packages/backend/nats/test/dkv.test.ts +++ b/src/packages/backend/nats/test/dkv.test.ts @@ -38,21 +38,33 @@ describe("create a public dkv and do basic operations", () => { }); }); -describe("opens a dkv twice and verifies it was cached", () => { +describe("opens a dkv twice and verifies the cached works and is reference counted", () => { let kv1; let kv2; const name = `test-${Math.random()}`; - it("creates the dkv twice", async () => { + it("creates the same dkv twice", async () => { kv1 = await createDkv({ name }); kv2 = await createDkv({ name }); expect(kv1.get()).toEqual({}); expect(kv1 === kv2).toBe(true); }); - it("closes", async () => { + + it("closes kv1 (one reference)", async () => { kv1.close(); + expect(kv2.get).not.toThrow(); + }); + + it("closes kv2 (another reference)", async () => { + kv2.close(); + // really closed! expect(kv2.get).toThrow("closed"); }); + + it("create and see it is new now", async () => { + kv1 = await createDkv({ name }); + expect(kv1 === kv2).toBe(false); + }); }); describe("opens a dkv twice at once and observe sync", () => { @@ -61,8 +73,8 @@ describe("opens a dkv twice at once and observe sync", () => { const name = `test-${Math.random()}`; it("creates the dkv twice", async () => { - kv1 = await createDkv({ name }, { noCache: true }); - kv2 = await createDkv({ name }, { noCache: true }); + kv1 = await createDkv({ name, noCache: true }); + kv2 = await createDkv({ name, noCache: true }); expect(kv1.get()).toEqual({}); expect(kv2.get()).toEqual({}); expect(kv1 === kv2).toBe(false); @@ -129,8 +141,8 @@ describe("test deleting and clearing a dkv", () => { const reset = async () => { const name = `test-${Math.random()}`; - kv1 = await createDkv({ name }, { noCache: true }); - kv2 = await createDkv({ name }, { noCache: true }); + kv1 = await createDkv({ name, noCache: true }); + kv2 = await createDkv({ name, noCache: true }); }; it("creates the dkv twice without caching so can make sure sync works", async () => { @@ -257,7 +269,7 @@ describe("create many distinct clients at once, write to all of them, and see th it(`creates the ${count} clients`, async () => { for (let i = 0; i < count; i++) { - clients[i] = await createDkv({ name }, { noCache: true }); + clients[i] = await createDkv({ name, noCache: true }); } }); @@ -301,8 +313,8 @@ describe("tests involving null/undefined values", () => { const name = `test-${Math.random()}`; it("creates the dkv twice", async () => { - kv1 = await createDkv({ name, noAutosave: true }, { noCache: true }); - kv2 = await createDkv({ name, noAutosave: true }, { noCache: true }); + kv1 = await createDkv({ name, noAutosave: true, noCache: true }); + kv2 = await createDkv({ name, noAutosave: true, noCache: true }); expect(kv1.get()).toEqual({}); expect(kv1 === kv2).toBe(false); }); diff --git a/src/packages/backend/nats/test/dstream.test.ts b/src/packages/backend/nats/test/dstream.test.ts index 34d2b4caf9..17a442217d 100644 --- a/src/packages/backend/nats/test/dstream.test.ts +++ b/src/packages/backend/nats/test/dstream.test.ts @@ -63,8 +63,8 @@ describe("create two dstreams and observe sync between them", () => { const name = `test-${Math.random()}`; let s1, s2; it("creates two distinct dstream objects s1 and s2 with the same name", async () => { - s1 = await createDstream({ name, noAutosave: true }, { noCache: true }); - s2 = await createDstream({ name, noAutosave: true }, { noCache: true }); + s1 = await createDstream({ name, noAutosave: true, noCache: true }); + s2 = await createDstream({ name, noAutosave: true, noCache: true }); // definitely distinct expect(s1 === s2).toBe(false); }); diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 533c6e1812..49cdd76de0 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -32,6 +32,7 @@ import Cookies from "js-cookie"; import { basePathCookieName } from "@cocalc/util/misc"; import { ACCOUNT_ID_COOKIE_NAME } from "@cocalc/util/db-schema/accounts"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; +import type { NatsSyncTableFunction } from "@cocalc/nats/sync/synctable"; // This DEBUG variable comes from webpack: declare const DEBUG; @@ -83,7 +84,7 @@ export interface WebappClient extends EventEmitter { get_username: Function; is_signed_in: () => boolean; synctable_project: Function; - synctable_nats: Function; + synctable_nats: NatsSyncTableFunction; pubsub_nats: Function; project_websocket: Function; prettier: Function; @@ -165,7 +166,7 @@ class Client extends EventEmitter implements WebappClient { get_username: Function; is_signed_in: () => boolean; synctable_project: Function; - synctable_nats: Function; + synctable_nats: NatsSyncTableFunction; pubsub_nats: Function; project_websocket: Function; prettier: Function; diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index fb597a66b5..e7d2c0b4db 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -4,7 +4,11 @@ import type { WebappClient } from "@cocalc/frontend/client/client"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { join } from "path"; import * as jetstream from "@nats-io/jetstream"; -import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; +import { + createSyncTable, + type NatsSyncTable, + NatsSyncTableFunction, +} from "@cocalc/nats/sync/synctable"; import { inboxPrefix, randomId } from "@cocalc/nats/names"; import { browserSubject, projectSubject } from "@cocalc/nats/names"; import { parse_query } from "@cocalc/sync/table/util"; @@ -250,54 +254,30 @@ export class NatsClient { }; }; - private synctableCache: { [key: string]: SyncTable } = {}; - synctable = reuseInFlight( - async ( - query, - options?: { - obj?: object; - atomic?: boolean; - immutable?: boolean; - stream?: boolean; - pubsub?: boolean; - throttleChanges?: number; - // for tables specific to a project, e.g., syncstrings in a project - project_id?: string; - }, - ): Promise => { - query = parse_query(query); - const key = JSON.stringify({ query, options }); - if (this.synctableCache[key] != null) { - return this.synctableCache[key]; - } - const table = keys(query)[0]; - const obj = options?.obj; - if (obj != null) { - for (const k in obj) { - query[table][0][k] = obj[k]; - } + synctable: NatsSyncTableFunction = async ( + query, + options?, + ): Promise => { + query = parse_query(query); + const table = keys(query)[0]; + const obj = options?.obj; + if (obj != null) { + for (const k in obj) { + query[table][0][k] = obj[k]; } - if ( - options?.project_id != null && - query[table][0]["project_id"] === null - ) { - query[table][0]["project_id"] = options.project_id; - } - const s = createSyncTable({ - ...options, - query, - env: await this.getEnv(), - account_id: this.client.account_id, - }); - this.synctableCache[key] = s; - // @ts-ignore - s.on("closed", () => { - delete this.synctableCache[key]; - }); - await s.init(); - return s; - }, - ); + } + if (options?.project_id != null && query[table][0]["project_id"] === null) { + query[table][0]["project_id"] = options.project_id; + } + const s = createSyncTable({ + ...options, + query, + env: await this.getEnv(), + account_id: this.client.account_id, + }); + await s.init(); + return s; + }; changefeedInterest = async (query, noError?: boolean) => { // express interest @@ -389,35 +369,35 @@ export class NatsClient { return accumulate; }; - stream = async (opts: Partial) => { + stream = async (opts: Partial & { name: string }) => { if (!opts.account_id && !opts.project_id && opts.limits != null) { throw Error("account client can't set limits on public stream"); } return await stream({ env: await this.getEnv(), ...opts }); }; - dstream = async (opts: Partial) => { + dstream = async (opts: Partial & { name: string }) => { if (!opts.account_id && !opts.project_id && opts.limits != null) { throw Error("account client can't set limits on public stream"); } return await dstream({ env: await this.getEnv(), ...opts }); }; - kv = async (opts: Partial) => { + kv = async (opts: Partial & { name: string }) => { // if (!opts.account_id && !opts.project_id && opts.limits != null) { // throw Error("account client can't set limits on public stream"); // } return await kv({ env: await this.getEnv(), ...opts }); }; - dkv = async (opts: Partial) => { + dkv = async (opts: Partial & { name: string }) => { // if (!opts.account_id && !opts.project_id && opts.limits != null) { // throw Error("account client can't set limits on public stream"); // } return await dkv({ env: await this.getEnv(), ...opts }); }; - dko = async (opts: Partial) => { + dko = async (opts: Partial & { name: string }) => { // if (!opts.account_id && !opts.project_id && opts.limits != null) { // throw Error("account client can't set limits on public stream"); // } diff --git a/src/packages/nats/sync/dko.ts b/src/packages/nats/sync/dko.ts index 15daead6e8..0037657a94 100644 --- a/src/packages/nats/sync/dko.ts +++ b/src/packages/nats/sync/dko.ts @@ -18,6 +18,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { dkv as createDKV, DKV, DKVOptions } from "./dkv"; import { userKvKey } from "./kv"; import { is_object } from "@cocalc/util/misc"; +import refCache from "@cocalc/util/refcache"; export interface DKOOptions extends DKVOptions { sep?: string; @@ -40,7 +41,7 @@ export class DKO extends EventEmitter { }, set(target, prop, value) { prop = String(prop); - if (prop == "_eventsCount" || prop == "_events") { + if (prop == "_eventsCount" || prop == "_events" || prop == "close") { target[prop] = value; return true; } @@ -133,6 +134,10 @@ export class DKO extends EventEmitter { } }; + clear = () => { + this.dkv?.clear(); + }; + get = (key?) => { if (this.dkv == null) { throw Error("closed"); @@ -209,22 +214,14 @@ export class DKO extends EventEmitter { }; } -const cache: { [key: string]: DKO } = {}; -export const dko = reuseInFlight( - async (opts: DKOOptions) => { - const key = userKvKey(opts); - if (cache[key] == null) { - const k = new DKO(opts); - await k.init(); - k.on("closed", () => delete cache[key]); - cache[key] = k; - } - return cache[key]!; - }, - { - createKey: (args) => userKvKey(args[0]), +export const dko = refCache({ + createKey: userKvKey, + createObject: async (opts) => { + const k = new DKO(opts); + await k.init(); + return k; }, -); +}); function dkoPrefix(name: string): string { return `__dko__${name}`; diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index 74992faa20..dbb048ed98 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -21,9 +21,10 @@ import { GeneralDKV, TOMBSTONE, type MergeFunction } from "./general-dkv"; import { userKvKey, type KVOptions } from "./kv"; import { jsName } from "@cocalc/nats/names"; import { sha1 } from "@cocalc/util/misc"; +import refCache from "@cocalc/util/refcache"; export interface DKVOptions extends KVOptions { - merge: MergeFunction; + merge?: MergeFunction; noAutosave?: boolean; } @@ -69,7 +70,7 @@ export class DKV extends EventEmitter { }, set(target, prop, value) { prop = String(prop); - if (prop == "_eventsCount" || prop == "_events") { + if (prop == "_eventsCount" || prop == "_events" || prop == "close") { target[prop] = value; return true; } @@ -251,27 +252,11 @@ export class DKV extends EventEmitter { }; } -const cache: { [key: string]: DKV } = {}; -export const dkv = reuseInFlight( - async (opts: DKVOptions, { noCache }: { noCache?: boolean } = {}) => { - const f = async () => { - const k = new DKV(opts); - await k.init(); - return k; - }; - if (noCache) { - // especially useful for unit testing. - return await f(); - } - const key = userKvKey(opts); - if (cache[key] == null) { - const k = await f(); - k.on("closed", () => delete cache[key]); - cache[key] = k; - } - return cache[key]!; - }, - { - createKey: (args) => userKvKey(args[0]) + JSON.stringify(args[1]), +export const dkv = refCache({ + createKey: userKvKey, + createObject: async (opts) => { + const k = new DKV(opts); + await k.init(); + return k; }, -); +}); diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 9ee739e81b..505e19fed2 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -31,6 +31,7 @@ import { map as awaitMap } from "awaiting"; import { isNumericString } from "@cocalc/util/misc"; import { sha1 } from "@cocalc/util/misc"; import { millis } from "@cocalc/nats/util"; +import refCache from "@cocalc/util/refcache"; const MAX_PARALLEL = 250; @@ -230,45 +231,22 @@ export class DStream extends EventEmitter { }; } -const dstreamCache: { [key: string]: DStream } = {}; -export const dstream = reuseInFlight( - async ( - options: UserStreamOptions, - { noCache }: { noCache?: boolean } = {}, - ) => { +export const dstream = refCache({ + createKey: userStreamOptionsKey, + createObject: async (options) => { const { account_id, project_id, name } = options; const jsname = jsName({ account_id, project_id }); const subjects = streamSubject({ account_id, project_id }); const filter = subjects.replace(">", (options.env.sha1 ?? sha1)(name)); - const f = async () => { - const dstream = new DStream({ - ...options, - name, - jsname, - subjects, - subject: filter, - filter, - }); - await dstream.init(); - return dstream; - }; - if (noCache) { - // especially useful for unit testing. - return await f(); - } - - const key = userStreamOptionsKey(options); - if (dstreamCache[key] == null) { - const dstream = await f(); - dstreamCache[key] = dstream; - dstream.on("closed", () => { - delete dstreamCache[key]; - }); - } - return dstreamCache[key]; - }, - { - createKey: (args) => - userStreamOptionsKey(args[0]) + JSON.stringify(args[1]), + const dstream = new DStream({ + ...options, + name, + jsname, + subjects, + subject: filter, + filter, + }); + await dstream.init(); + return dstream; }, -); +}); diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index 085eee82e9..00715320aa 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -163,7 +163,7 @@ export class GeneralKV extends EventEmitter { filter?: string | string[]; env: NatsEnv; options?; - limits?: KVLimits; + limits?: Partial; }) { super(); this.limits = { diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index c9fcf2848c..b698cd2e9a 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -19,13 +19,15 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { GeneralKV, type KVLimits } from "./general-kv"; import { jsName } from "@cocalc/nats/names"; import { sha1 } from "@cocalc/util/misc"; +import refCache from "@cocalc/util/refcache"; export interface KVOptions { name: string; account_id?: string; project_id?: string; env: NatsEnv; - limits?: KVLimits; + limits?: Partial; + noCache?: boolean; } export class KV extends EventEmitter { @@ -149,19 +151,12 @@ export function userKvKey(options: KVOptions) { return JSON.stringify(x); } -const cache: { [key: string]: KV } = {}; -export const kv = reuseInFlight( - async (opts: KVOptions) => { - const key = userKvKey(opts); - if (cache[key] == null) { - const k = new KV(opts); - await k.init(); - k.on("closed", () => delete cache[key]); - cache[key] = k; - } - return cache[key]!; - }, - { - createKey: (args) => userKvKey(args[0]), + +export const kv = refCache({ + createKey: userKvKey, + createObject: async (opts) => { + const k = new KV(opts); + await k.init(); + return k; }, -); +}); diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index f6fff669a5..9f222445f1 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -85,18 +85,16 @@ export class OpenFiles extends EventEmitter { }; init = async () => { - const d = await dkv( - { - name: "open-files", - project_id: this.project_id, - env: this.env, - limits: { - max_age: nanos(MAX_AGE_MS), - }, - noAutosave: this.noAutosave, + const d = await dkv({ + name: "open-files", + project_id: this.project_id, + env: this.env, + limits: { + max_age: nanos(MAX_AGE_MS), }, - { noCache: this.noCache }, - ); + noAutosave: this.noAutosave, + noCache: this.noCache, + }); this.dkv = d; d.on("change", ({ key: path }) => { const entry = this.get(path); diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 3e1f572856..89707cb368 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -49,6 +49,7 @@ import { throttle } from "lodash"; import { isNumericString } from "@cocalc/util/misc"; import { map as awaitMap } from "awaiting"; import { sha1 } from "@cocalc/util/misc"; +import refCache from "@cocalc/util/refcache"; class PublishRejectError extends Error { code: string; @@ -557,6 +558,7 @@ export interface UserStreamOptions { project_id?: string; limits?: FilteredStreamLimitOptions; start_seq?: number; + noCache?: boolean; } export function userStreamOptionsKey(options: UserStreamOptions) { @@ -567,32 +569,22 @@ export function userStreamOptionsKey(options: UserStreamOptions) { return JSON.stringify(x); } -const streamCache: { [key: string]: Stream } = {}; -export const stream = reuseInFlight( - async (options: UserStreamOptions) => { +export const stream = refCache({ + createKey: userStreamOptionsKey, + createObject: async (options) => { const { account_id, project_id, name } = options; const jsname = jsName({ account_id, project_id }); const subjects = streamSubject({ account_id, project_id }); const filter = subjects.replace(">", (options.env.sha1 ?? sha1)(name)); - const key = userStreamOptionsKey(options); - if (streamCache[key] == null) { - const stream = new Stream({ - ...options, - name, - jsname, - subjects, - subject: filter, - filter, - }); - await stream.init(); - streamCache[key] = stream; - stream.on("closed", () => { - delete streamCache[key]; - }); - } - return streamCache[key]; - }, - { - createKey: (args) => userStreamOptionsKey(args[0]), + const stream = new Stream({ + ...options, + name, + jsname, + subjects, + subject: filter, + filter, + }); + await stream.init(); + return stream; }, -); +}); diff --git a/src/packages/nats/sync/syncdoc-info.ts b/src/packages/nats/sync/syncdoc-info.ts new file mode 100644 index 0000000000..042c4d4ee9 --- /dev/null +++ b/src/packages/nats/sync/syncdoc-info.ts @@ -0,0 +1,48 @@ +import { client_db } from "@cocalc/util/db-schema/client-db"; + +export async function getSyncDocType({ + client, + project_id, + path, +}): Promise<{ type: "db" | "string"; opts?: any }> { + // instead of just "querying the db" (i.e., nats in this case), + // we create the synctable. This avoids race conditions, since we + // can wait until data is written, and also abstracts away the + // internal structure. + let syncdocs; + try { + const string_id = client_db.sha1(project_id, path); + syncdocs = await client.synctable_nats( + { syncstrings: [{ project_id, path, string_id, doctype: null }] }, + { + stream: false, + atomic: false, + immutable: false, + }, + ); + let s = syncdocs.get_one(); + if (s?.doctype == null) { + // wait until there is a syncstring and its doctype is set: + await syncdocs.wait(() => { + s = syncdocs.get_one(); + return s?.doctype != null; + }, 10); + } + let doctype; + try { + doctype = JSON.parse(s.doctype); + } catch (err) { + console.warn("malformed doctype", err); + doctype = { type: "string" }; + } + if (doctype.type !== "db" && doctype.type !== "string") { + // ensure valid type + console.warn("invalid docstype", doctype.type); + doctype.type = "string"; + } + return doctype; + } finally { + // be sure to close this no matter what, since no value in watching changes. + syncdocs?.close(); + } +} diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/nats/sync/synctable.ts index bef9d83eba..770c78c33e 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/nats/sync/synctable.ts @@ -1,23 +1,29 @@ import { type NatsEnv } from "@cocalc/nats/types"; import { SyncTableKV } from "./synctable-kv"; import { SyncTableStream } from "./synctable-stream"; +import { refCacheSync } from "@cocalc/util/refcache"; -export type SyncTable = SyncTableStream | SyncTableKV; +export type NatsSyncTable = SyncTableStream | SyncTableKV; + +export type NatsSyncTableFunction = ( + query: { [table: string]: { [field: string]: any }[] }, + options?: { + obj?: object; + atomic?: boolean; + immutable?: boolean; + stream?: boolean; + pubsub?: boolean; + throttleChanges?: number; + // for tables specific to a project, e.g., syncstrings in a project + project_id?: string; + }, +) => Promise; // When the database is watching tables for changefeeds, if it doesn't get a clear expression // of interest from a client every this much time, it automatically stops. export const CHANGEFEED_INTEREST_PERIOD_MS = 120000; -export function createSyncTable({ - query, - env, - account_id, - project_id, - atomic, - stream, - immutable, - ...options -}: { +interface Options { query; env: NatsEnv; account_id?: string; @@ -25,25 +31,18 @@ export function createSyncTable({ atomic?: boolean; stream?: boolean; immutable?: boolean; // if true, then get/set works with immutable.js objects instead. -}) { - if (stream) { - return new SyncTableStream({ - query, - env, - account_id, - project_id, - immutable, - ...options, - }); + noCache?: boolean; +} + +function createObject(options: Options) { + if (options.stream) { + return new SyncTableStream(options); } else { - return new SyncTableKV({ - query, - env, - account_id, - project_id, - atomic, - immutable, - ...options, - }); + return new SyncTableKV(options); } } + +export const createSyncTable = refCacheSync({ + createKey: (opts) => JSON.stringify({ ...opts, env: undefined }), + createObject, +}); diff --git a/src/packages/nats/types.ts b/src/packages/nats/types.ts index b981f18437..c09afecb2a 100644 --- a/src/packages/nats/types.ts +++ b/src/packages/nats/types.ts @@ -6,4 +6,3 @@ export interface NatsEnv { } export type State = "disconnected" | "connected" | "closed"; - diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 4599d6ac6f..bc235342d4 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1902,6 +1902,9 @@ importers: sync: dependencies: + '@cocalc/nats': + specifier: workspace:* + version: link:../nats '@cocalc/sync': specifier: workspace:* version: 'link:' diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index 4fc7685c00..c9b4f57b8b 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -50,6 +50,7 @@ import { get_synctable } from "./sync/open-synctables"; import { get_syncdoc } from "./sync/sync-doc"; import synctable_nats from "@cocalc/project/nats/synctable"; import pubsub from "@cocalc/project/nats/pubsub"; +import type { NatsSyncTableFunction } from "@cocalc/nats/sync/synctable"; const winston = getLogger("client"); @@ -507,7 +508,7 @@ export class Client extends EventEmitter implements ProjectClientInterface { return the_synctable; } - synctable_nats = async (query, options?) => { + synctable_nats: NatsSyncTableFunction = async (query, options?) => { return await synctable_nats(query, options); }; diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 67361e2288..27d233cdd1 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -6,6 +6,8 @@ DEVELOPMENT: 0. From the browser, terminate open-files api service running in the project already, if any await cc.client.nats_client.projectApi({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}).system.terminate({service:'open-files'}) + + // {status: 'terminated', service: 'open-files'} @@ -31,6 +33,7 @@ import { type OpenFiles, type OpenFileEntry, } from "@cocalc/project/nats/sync"; +import { getSyncDocType } from "@cocalc/nats/sync/syncdoc-info"; import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; import { /*compute_server_id,*/ project_id } from "@cocalc/project/data"; import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; @@ -40,7 +43,6 @@ import { SyncDB } from "@cocalc/sync/editor/db/sync"; import getLogger from "@cocalc/backend/logger"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { delay } from "awaiting"; -import { client_db } from "@cocalc/util/db-schema"; const logger = getLogger("project:nats:open-files"); @@ -142,15 +144,15 @@ async function closeIgnoredFilesLoop() { "currently open paths...", ); const cutoff = getCutoff(); - for (const path of paths) { - const entry = openFiles.get(path); + for (const entry of openFiles.getAll()) { if ( + entry != null && entry.time != null && entry.time <= cutoff && - supportAutoclose(path) + supportAutoclose(entry.path) ) { - logger.debug("closeIgnoredFiles: closing due to inactivity", { path }); - closeSyncDoc(path); + logger.debug("closeIgnoredFiles: closing due to inactivity", entry); + closeSyncDoc(entry.path); } } } @@ -179,36 +181,24 @@ const openSyncDoc = reuseInFlight(async (path: string) => { return; } const client = getClient(); - let x; - try { - const string_id = client_db.sha1(project_id, path); - const syncstrings = await client.synctable_nats( - { syncstrings: [{ string_id, doctype: null }] }, - { - stream: false, - atomic: false, - immutable: false, - }, - ); - x = await getTypeAndOpts(syncstrings); - } catch (err) { - logger.debug(`openSyncDoc failed - error = ${err}`); - return; - } - const { type, opts } = x; - logger.debug("openSyncDoc got", { path, type, opts }); + const doctype = await getSyncDocType({ + project_id, + path, + client, + }); + logger.debug("openSyncDoc got", { path, doctype }); let doc; - if (type == "string") { + if (doctype.type == "string") { doc = new SyncString({ - ...opts, + ...doctype.opts, project_id, path, client, }); } else { doc = new SyncDB({ - ...opts, + ...doctype.opts, project_id, path, client, @@ -217,40 +207,3 @@ const openSyncDoc = reuseInFlight(async (path: string) => { openSyncDocs[path] = doc; return; }); - -async function getTypeAndOpts( - syncstrings, -): Promise<{ type: string; opts: any }> { - // global.z = { syncstrings }; - let s = syncstrings.get_one(); - if (s?.doctype == null) { - // wait until there is a syncstring and its doctype is set: - await syncstrings.wait(() => { - s = syncstrings.get_one(); - return s?.doctype != null; - }); - } - const opts: any = {}; - let type: string = ""; - - let doctype = s.doctype; - if (doctype != null) { - try { - doctype = JSON.parse(doctype); - } catch { - doctype = {}; - } - if (doctype.opts != null) { - for (const k in doctype.opts) { - opts[k] = doctype.opts[k]; - } - } - type = doctype.type; - } - opts.doctype = doctype; - if (type !== "db" && type !== "string") { - // fallback type - type = "string"; - } - return { type, opts }; -} diff --git a/src/packages/project/nats/synctable.ts b/src/packages/project/nats/synctable.ts index 9a9f77b691..0eeb90250a 100644 --- a/src/packages/project/nats/synctable.ts +++ b/src/packages/project/nats/synctable.ts @@ -2,40 +2,38 @@ import getConnection from "./connection"; import { project_id } from "@cocalc/project/data"; import { JSONCodec } from "nats"; import { sha1 } from "@cocalc/backend/sha1"; -import { createSyncTable, type SyncTable } from "@cocalc/nats/sync/synctable"; +import { + createSyncTable, + type NatsSyncTable, +} from "@cocalc/nats/sync/synctable"; import { parse_query } from "@cocalc/sync/table/util"; import { keys } from "lodash"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import type { NatsSyncTableFunction } from "@cocalc/nats/sync/synctable"; const jc = JSONCodec(); -const cache: { [key: string]: SyncTable } = {}; -const synctable = reuseInFlight(async (query, options?) => { - const key = JSON.stringify(query); - if (cache[key] == null) { - const nc = await getConnection(); - query = parse_query(query); - const table = keys(query)[0]; - const obj = options?.obj; - if (obj != null) { - for (const k in obj) { - query[table][0][k] = obj[k]; - } +const synctable: NatsSyncTableFunction = async ( + query, + options?, +): Promise => { + const nc = await getConnection(); + query = parse_query(query); + const table = keys(query)[0]; + const obj = options?.obj; + if (obj != null) { + for (const k in obj) { + query[table][0][k] = obj[k]; } - query[table][0].project_id = project_id; - const s = createSyncTable({ - project_id, - ...options, - query, - env: { sha1, jc, nc }, - }); - await s.init(); - cache[key] = s; - s.on("closed", () => { - delete cache[key]; - }); } - return cache[key]; -}); + query[table][0].project_id = project_id; + const s = createSyncTable({ + project_id, + ...options, + query, + env: { sha1, jc, nc }, + }); + await s.init(); + return s; +}; export default synctable; diff --git a/src/packages/sync/client/sync-client.ts b/src/packages/sync/client/sync-client.ts index e6a1aee4d6..36cba23546 100644 --- a/src/packages/sync/client/sync-client.ts +++ b/src/packages/sync/client/sync-client.ts @@ -7,14 +7,8 @@ Functionality related to Sync. */ -import { callback2 } from "@cocalc/util/async-utils"; import { once } from "@cocalc/util/async-utils"; -import { - defaults, - is_valid_uuid_string, - merge, - required, -} from "@cocalc/util/misc"; +import { defaults, is_valid_uuid_string, required } from "@cocalc/util/misc"; import { SyncDoc, SyncOpts0 } from "@cocalc/sync/editor/generic/sync-doc"; import { SyncDB, SyncDBOpts0 } from "@cocalc/sync/editor/db"; import { SyncString } from "@cocalc/sync/editor/string/sync"; @@ -27,6 +21,7 @@ import { } from "@cocalc/sync/table"; import synctable_project from "./synctable-project"; import type { Channel, AppClient } from "./types"; +import { getSyncDocType } from "@cocalc/nats/sync/syncdoc-info"; interface SyncOpts extends Omit {} @@ -142,44 +137,30 @@ export class SyncClient { return new SyncDB(opts0); } - public async open_existing_sync_document(opts: { + public async open_existing_sync_document({ + project_id, + path, + data_server, + persistent, + }: { project_id: string; path: string; data_server?: string; persistent?: boolean; }): Promise { - const resp = await callback2(this.client.query, { - query: { - syncstrings: { - project_id: opts.project_id, - path: opts.path, - doctype: null, - }, - }, + const doctype = await getSyncDocType({ + project_id, + path, + client: this.client, + }); + const { type } = doctype; + const f = `sync_${type}`; + return (this as any)[f]({ + project_id, + path, + data_server, + persistent, + ...doctype.opts, }); - if (resp.event === "error") { - throw Error(resp.error); - } - if (resp.query?.syncstrings == null) { - throw Error(`no document '${opts.path}' in project '${opts.project_id}'`); - } - const doctype = JSON.parse( - resp.query.syncstrings.doctype ?? '{"type":"string"}', - ); - let opts2: any = { - project_id: opts.project_id, - path: opts.path, - }; - if (opts.data_server) { - opts2.data_server = opts.data_server; - } - if (opts.persistent) { - opts2.persistent = opts.persistent; - } - if (doctype.opts != null) { - opts2 = merge(opts2, doctype.opts); - } - const f = `sync_${doctype.type}`; - return (this as any)[f](opts2); } } diff --git a/src/packages/sync/package.json b/src/packages/sync/package.json index 2907733a2c..3ce115b205 100644 --- a/src/packages/sync/package.json +++ b/src/packages/sync/package.json @@ -15,21 +15,14 @@ "test": "pnpm exec jest", "prepublishOnly": "pnpm test" }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "cocalc", - "realtime synchronization" - ], + "keywords": ["cocalc", "realtime synchronization"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", + "@cocalc/nats": "workspace:*", "@types/debug": "^4.1.12", "@types/lodash": "^4.14.202", "async": "^1.5.2", diff --git a/src/packages/sync/tsconfig.json b/src/packages/sync/tsconfig.json index 6cdb913e19..4cdbe9694e 100644 --- a/src/packages/sync/tsconfig.json +++ b/src/packages/sync/tsconfig.json @@ -5,5 +5,5 @@ "outDir": "dist" }, "exclude": ["node_modules", "dist", "test"], - "references": [{ "path": "../util" }] + "references": [{ "path": "../util" }, { "path": "../nats" }] } diff --git a/src/packages/util/db-schema/syncstring-schema.ts b/src/packages/util/db-schema/syncstring-schema.ts index 09f18cf3e7..e4bd8b94a2 100644 --- a/src/packages/util/db-schema/syncstring-schema.ts +++ b/src/packages/util/db-schema/syncstring-schema.ts @@ -66,7 +66,7 @@ Table({ }, doctype: { type: "string", - desc: "(optional) JSON string describing meaning of the patches (i.e., of this document0 -- e.g., {type:'db', opts:{primary_keys:['id'], string_cols:['name']}}", + desc: "(REQUIRED) JSON string describing meaning of the patches (i.e., of this document0 -- e.g., {type:'db', opts:{primary_keys:['id'], string_cols:['name']}}", }, settings: { type: "map", diff --git a/src/packages/util/refcache.ts b/src/packages/util/refcache.ts new file mode 100644 index 0000000000..a28cdfde74 --- /dev/null +++ b/src/packages/util/refcache.ts @@ -0,0 +1,111 @@ +/* +A reference counting cache. + +See example usage in nats/sync. +*/ + +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; + +export default function refCache< + Options extends { noCache?: boolean }, + T extends { close: () => void }, +>({ + createKey, + createObject, +}: { + createKey?: (opts: Options) => string; + createObject: (opts: Options) => Promise; +}) { + const cache: { [key: string]: T } = {}; + const count: { [key: number]: T } = {}; + const close: { [key: number]: Function } = {}; + if (createKey == null) { + createKey = JSON.stringify; + } + const createObjectReuseInFlight = reuseInFlight(createObject, { + createKey: (args) => createKey(args[0]), + }); + + const get = async (opts: Options): Promise => { + if (opts.noCache) { + return await createObject(opts); + } + const key = createKey(opts); + if (cache[key] != undefined) { + count[key] += 1; + return cache[key]; + } + const obj = await createObjectReuseInFlight(opts); + if (cache[key] != null) { + // it's possible after the above await that a + // different call to get already setup the cache, count, etc. + count[key] += 1; + return cache[key]; + } + // we are *the* one setting things up. + cache[key] = obj; + count[key] = 1; + close[key] = obj.close; + obj.close = () => { + count[key] -= 1; + if (count[key] <= 0) { + obj.close = close[key]; + obj.close?.(); + delete cache[key]; + delete count[key]; + delete close[key]; + } + }; + + return obj; + }; + + return get; +} + +export function refCacheSync< + Options extends { noCache?: boolean }, + T extends { close: () => void }, +>({ + createKey, + createObject, +}: { + createKey?: (opts: Options) => string; + createObject: (opts: Options) => T; +}) { + const cache: { [key: string]: T } = {}; + const count: { [key: number]: T } = {}; + const close: { [key: number]: Function } = {}; + if (createKey == null) { + createKey = JSON.stringify; + } + const get = (opts: Options): T => { + if (opts.noCache) { + return createObject(opts); + } + const key = createKey(opts); + if (cache[key] != undefined) { + count[key] += 1; + return cache[key]; + } + const obj = createObject(opts); + // we are *the* one setting things up. + cache[key] = obj; + count[key] = 1; + close[key] = obj.close; + obj.close = () => { + count[key] -= 1; + if (count[key] <= 0) { + obj.close = close[key]; + obj.close?.(); + delete cache[key]; + delete count[key]; + delete close[key]; + } + }; + + return obj; + }; + + return get; +} From db264ec845611d11369b149966e7fade139bbbbc Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 14 Feb 2025 00:51:40 +0000 Subject: [PATCH 192/281] nats + trimetravel -- make timetravel close syncdoc it opens - has some bad invisible side effects if you also open same timetravel in a tab... but who would do that? --- .../frame-editors/frame-tree/frame-tree.tsx | 2 +- .../frame-editors/frame-tree/register.ts | 1 + .../time-travel-editor/actions.ts | 8 +++++ .../time-travel-editor/register.ts | 1 - src/packages/nats/sync/synctable.ts | 1 + src/packages/sync/editor/generic/sync-doc.ts | 20 ++++++++++++- src/packages/util/refcache.ts | 30 +++++++++++++++++++ 7 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/packages/frontend/frame-editors/frame-tree/frame-tree.tsx b/src/packages/frontend/frame-editors/frame-tree/frame-tree.tsx index 573810972b..3eecac3afa 100644 --- a/src/packages/frontend/frame-editors/frame-tree/frame-tree.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/frame-tree.tsx @@ -279,7 +279,7 @@ export const FrameTree: React.FC = React.memo( // UGLY/TODO: This approach to TimeTravel as a frame is not sufficiently // generic and is a **temporary** hack. It'll be rewritten - // soon in a more generic way that also will support multifile + // someday in a more generic way that also will support multifile // latex editing. See https://github.com/sagemathinc/cocalc/issues/904 // Note that this does NOT reference count the actions properly // right now... We need to switch to something like we do with diff --git a/src/packages/frontend/frame-editors/frame-tree/register.ts b/src/packages/frontend/frame-editors/frame-tree/register.ts index 3a73185146..b1d10c3251 100644 --- a/src/packages/frontend/frame-editors/frame-tree/register.ts +++ b/src/packages/frontend/frame-editors/frame-tree/register.ts @@ -126,6 +126,7 @@ function register( .getProjectStore(project_id) ?.getIn(["open_files", actions.timeTravelActions.path]) ) { + actions.timeTravelActions.close(); redux.removeActions(actions.timeTravelActions.name); redux.removeStore(actions.timeTravelActions.name); } diff --git a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts index 554b4da6ce..b8873e25fe 100644 --- a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts +++ b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts @@ -109,6 +109,14 @@ export class TimeTravelActions extends CodeEditorActions { return { type: "time_travel" }; } + close(): void { + if (this.syncdoc != null) { + this.syncdoc.close(); + delete this.syncdoc; + } + super.close(); + } + set_error = (error) => { this.setState({ error }); }; diff --git a/src/packages/frontend/frame-editors/time-travel-editor/register.ts b/src/packages/frontend/frame-editors/time-travel-editor/register.ts index f3bca216e4..e575943461 100644 --- a/src/packages/frontend/frame-editors/time-travel-editor/register.ts +++ b/src/packages/frontend/frame-editors/time-travel-editor/register.ts @@ -9,7 +9,6 @@ Register the TimeTravel frame tree editor import { Editor } from "./editor"; import { TimeTravelActions } from "./actions"; - import { register_file_editor } from "../frame-tree/register"; register_file_editor({ diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/nats/sync/synctable.ts index 770c78c33e..279c2099d1 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/nats/sync/synctable.ts @@ -45,4 +45,5 @@ function createObject(options: Options) { export const createSyncTable = refCacheSync({ createKey: (opts) => JSON.stringify({ ...opts, env: undefined }), createObject, + // name: "synctable-nats", }); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index e840fb835b..eff1f3cd2e 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1392,10 +1392,28 @@ export class SyncDoc extends EventEmitter { // Used for internal debug logging private dbg = (f: string = ""): Function => { + if (this.client == null) { + // dbg shouldn't be called after this synctable is closed, so in this + // case we clean up. There's a tricky cases involving timetravel + // in two tabs at once that could trigger this. + // In particular, the following breaks this: + // - open a.txt and timetravel in that frame + // - shift click the timetravel button to open another time travel tab + // - close the a.txt tab entirely + // - close timetravel (thus all tabs closed) + // - open a.txt again + // - type something and the patch update queue and this dbg gets triggered. + // The real problem as explained in frontend/frame-editors/frame-tree/frame-tree.tsx + // about timetravel is that it's hacky. + return () => {}; + // return (...args) => { + // console.warn(`BUG: use of a closed syncdoc -- SyncDoc.${f}`, ...args); + // }; + } // if (this.useNats) { // return (...args) => console.log(f, ...args); // } - return this.client?.dbg(`SyncDoc('${this.path}').${f}`); + return this.client.dbg(`SyncDoc('${this.path}').${f}`); }; private async init_all(): Promise { diff --git a/src/packages/util/refcache.ts b/src/packages/util/refcache.ts index a28cdfde74..1a20e88990 100644 --- a/src/packages/util/refcache.ts +++ b/src/packages/util/refcache.ts @@ -12,9 +12,11 @@ export default function refCache< >({ createKey, createObject, + name, }: { createKey?: (opts: Options) => string; createObject: (opts: Options) => Promise; + name?: string; }) { const cache: { [key: string]: T } = {}; const count: { [key: number]: T } = {}; @@ -33,9 +35,19 @@ export default function refCache< const key = createKey(opts); if (cache[key] != undefined) { count[key] += 1; + if (name) { + console.log("refCache: cache hit", { + name, + key, + count: count[key], + }); + } return cache[key]; } const obj = await createObjectReuseInFlight(opts); + if (name) { + console.log("refCache: create", { name, key }); + } if (cache[key] != null) { // it's possible after the above await that a // different call to get already setup the cache, count, etc. @@ -48,6 +60,9 @@ export default function refCache< close[key] = obj.close; obj.close = () => { count[key] -= 1; + if (name) { + console.log("refCache: close", { name, key, count: count[key] }); + } if (count[key] <= 0) { obj.close = close[key]; obj.close?.(); @@ -69,9 +84,11 @@ export function refCacheSync< >({ createKey, createObject, + name, }: { createKey?: (opts: Options) => string; createObject: (opts: Options) => T; + name?: string; }) { const cache: { [key: string]: T } = {}; const count: { [key: number]: T } = {}; @@ -86,15 +103,28 @@ export function refCacheSync< const key = createKey(opts); if (cache[key] != undefined) { count[key] += 1; + if (name) { + console.log("refCacheSync: cache hit", { + name, + key, + count: count[key], + }); + } return cache[key]; } const obj = createObject(opts); + if (name) { + console.log("refCacheSync: create", { name, key }); + } // we are *the* one setting things up. cache[key] = obj; count[key] = 1; close[key] = obj.close; obj.close = () => { count[key] -= 1; + if (name) { + console.log("refCacheSync: close", { name, key, count: count[key] }); + } if (count[key] <= 0) { obj.close = close[key]; obj.close?.(); From 6f30fa5e12f0871d19b1f6c7b8109680f7f2cddb Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 14 Feb 2025 03:29:27 +0000 Subject: [PATCH 193/281] nats: create nice abstraction for services --- src/packages/backend/nats/service.ts | 16 ++ .../backend/nats/test/service.test.ts | 35 +++++ src/packages/frontend/client/client.ts | 10 ++ src/packages/frontend/nats/client.ts | 25 +++- src/packages/jupyter/package.json | 13 +- src/packages/jupyter/redux/actions.ts | 67 ++------- src/packages/jupyter/tsconfig.json | 3 +- src/packages/nats/names.ts | 23 ++- src/packages/nats/package.json | 13 +- src/packages/nats/service.ts | 138 ++++++++++++++++++ src/packages/pnpm-lock.yaml | 6 + src/packages/project/client.ts | 15 ++ src/packages/project/nats/env.ts | 2 +- src/packages/project/nats/names.ts | 2 +- src/packages/server/nats/auth.ts | 1 - src/packages/sync/client/types.ts | 6 + src/packages/sync/editor/generic/sync-doc.ts | 10 ++ src/packages/sync/editor/generic/types.ts | 6 + .../sync/editor/string/test/client-test.ts | 4 + 19 files changed, 311 insertions(+), 84 deletions(-) create mode 100644 src/packages/backend/nats/service.ts create mode 100644 src/packages/backend/nats/test/service.test.ts create mode 100644 src/packages/nats/service.ts diff --git a/src/packages/backend/nats/service.ts b/src/packages/backend/nats/service.ts new file mode 100644 index 0000000000..7fca5653c6 --- /dev/null +++ b/src/packages/backend/nats/service.ts @@ -0,0 +1,16 @@ +import { + callNatsService as call, + createNatsService as create, +} from "@cocalc/nats/service"; +import type { + CallNatsServiceFunction, + CreateNatsServiceFunction, +} from "@cocalc/nats/service"; + +import { getEnv } from "@cocalc/backend/nats/env"; + +export const callNatsService: CallNatsServiceFunction = async (opts) => + await call({ ...opts, env: await getEnv() }); + +export const createNatsService: CreateNatsServiceFunction = async (opts) => + await create({ ...opts, env: await getEnv() }); diff --git a/src/packages/backend/nats/test/service.test.ts b/src/packages/backend/nats/test/service.test.ts new file mode 100644 index 0000000000..83e8246e83 --- /dev/null +++ b/src/packages/backend/nats/test/service.test.ts @@ -0,0 +1,35 @@ +/* + +DEVELOPMENT: + +pnpm exec jest --watch --forceExit --detectOpenHandles "service.test.ts" + +*/ + +import { + callNatsService, + createNatsService, +} from "@cocalc/backend/nats/service"; + +describe("create a service and test it out", () => { + let s; + it("creates a service", async () => { + s = await createNatsService({ service: "echo", handler: (mesg) => mesg }); + expect(await callNatsService({ service: "echo", mesg: "hello" })).toBe( + "hello", + ); + }); + it("closes the services", async () => { + s.close(); + + let t = ""; + // expect( ...).toThrow doesn't seem to work with this: + try { + await callNatsService({ service: "echo", mesg: "hi" }); + } catch (err) { + t = `${err}`; + } + expect(t).toContain("503"); + }); +}); + diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 49cdd76de0..f0b6008c77 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -33,6 +33,10 @@ import { basePathCookieName } from "@cocalc/util/misc"; import { ACCOUNT_ID_COOKIE_NAME } from "@cocalc/util/db-schema/accounts"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import type { NatsSyncTableFunction } from "@cocalc/nats/sync/synctable"; +import type { + CallNatsServiceFunction, + CreateNatsServiceFunction, +} from "@cocalc/nats/service"; // This DEBUG variable comes from webpack: declare const DEBUG; @@ -85,6 +89,8 @@ export interface WebappClient extends EventEmitter { is_signed_in: () => boolean; synctable_project: Function; synctable_nats: NatsSyncTableFunction; + callNatsService: CallNatsServiceFunction; + createNatsService: CreateNatsServiceFunction; pubsub_nats: Function; project_websocket: Function; prettier: Function; @@ -167,6 +173,8 @@ class Client extends EventEmitter implements WebappClient { is_signed_in: () => boolean; synctable_project: Function; synctable_nats: NatsSyncTableFunction; + callNatsService: CallNatsServiceFunction; + createNatsService: CreateNatsServiceFunction; pubsub_nats: Function; project_websocket: Function; prettier: Function; @@ -262,6 +270,8 @@ class Client extends EventEmitter implements WebappClient { ); this.synctable_nats = this.nats_client.synctable; this.pubsub_nats = this.nats_client.pubsub; + this.callNatsService = this.nats_client.callNatsService; + this.createNatsService = this.nats_client.createNatsService; this.query = this.query_client.query.bind(this.query_client); this.async_query = this.query_client.query.bind(this.query_client); diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index e7d2c0b4db..71204a3740 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -31,6 +31,11 @@ import { initApi } from "@cocalc/frontend/nats/api"; import { delay } from "awaiting"; import { Svcm } from "@nats-io/services"; import { CONNECT_OPTIONS } from "@cocalc/util/nats"; +import { callNatsService, createNatsService } from "@cocalc/nats/service"; +import type { + CallNatsServiceFunction, + CreateNatsServiceFunction, +} from "@cocalc/nats/service"; export class NatsClient { client: WebappClient; @@ -87,10 +92,18 @@ export class NatsClient { return this.nc; }); + callNatsService: CallNatsServiceFunction = async (options) => { + return await callNatsService({ ...options, env: await this.getEnv() }); + }; + + createNatsService: CreateNatsServiceFunction = async (options) => { + return createNatsService({ ...options, env: await this.getEnv() }); + }; + // deprecated! projectWebsocketApi = async ({ project_id, mesg, timeout = 5000 }) => { const nc = await this.getConnection(); - const subject = `${projectSubject({ project_id })}.browser-api`; + const subject = projectSubject({ project_id, service: "browser-api" }); const resp = await nc.request(subject, this.jc.encode(mesg), { timeout, }); @@ -162,13 +175,13 @@ export class NatsClient { }: { service?: string; project_id: string; - compute_server_id: number; + compute_server_id?: number; name: string; args: any[]; timeout?: number; }) => { const nc = await this.getConnection(); - const subject = `${projectSubject({ project_id, compute_server_id })}.${service}`; + const subject = projectSubject({ project_id, compute_server_id, service }); const mesg = this.jc.encode({ name, args, @@ -307,7 +320,11 @@ export class NatsClient { // DEPRECATED primus = async (project_id: string) => { return getPrimusConnection({ - subject: `${projectSubject({ project_id, compute_server_id: 0 })}.primus`, + subject: projectSubject({ + project_id, + compute_server_id: 0, + service: "primus", + }), env: await this.getEnv(), role: "client", id: this.sessionId, diff --git a/src/packages/jupyter/package.json b/src/packages/jupyter/package.json index 25e611a3ed..fee513ed28 100644 --- a/src/packages/jupyter/package.json +++ b/src/packages/jupyter/package.json @@ -24,17 +24,9 @@ "test": "pnpm exec jest", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput" }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "cocalc", - "jupyter" - ], + "keywords": ["cocalc", "jupyter"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", @@ -42,6 +34,7 @@ "@cocalc/sync": "workspace:*", "@cocalc/sync-client": "workspace:*", "@cocalc/util": "workspace:*", + "@cocalc/nats": "workspace:*", "@nteract/messaging": "^7.0.20", "@types/better-sqlite3": "^7.6.4", "@types/node-cleanup": "^2.1.2", diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index d5bccb0d07..18faba8ef9 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -20,7 +20,7 @@ import { Actions } from "@cocalc/util/redux/Actions"; import { three_way_merge } from "@cocalc/sync/editor/generic/util"; import { callback2, retry_until_success } from "@cocalc/util/async-utils"; import * as misc from "@cocalc/util/misc"; -import { callback, delay } from "awaiting"; +import { delay } from "awaiting"; import * as cell_utils from "@cocalc/jupyter/util/cell-utils"; import { JupyterStore, @@ -39,7 +39,6 @@ import { } from "@cocalc/jupyter/util/misc"; import { SyncDB } from "@cocalc/sync/editor/db/sync"; import type { Client } from "@cocalc/sync/client/types"; -import { once } from "@cocalc/util/async-utils"; import latexEnvs from "@cocalc/util/latex-envs"; const { close, required, defaults } = misc; @@ -190,66 +189,30 @@ export abstract class JupyterActions extends Actions { } }; - private apiCallHandler: { - id: number; // this is a sequential id used for request/response pairing - // when get response from computer server, one of these callbacks gets called: - responseCallbacks: { [id: number]: (err: any, response?: any) => void }; - } | null = null; - - private initApiCallHandler = () => { - this.apiCallHandler = { id: 0, responseCallbacks: {} }; - const { responseCallbacks } = this.apiCallHandler; - this.syncdb.on("message", (data) => { - const cb = responseCallbacks[data.id]; - if (cb != null) { - delete responseCallbacks[data.id]; - if (data.response?.event == "error") { - cb(data.response.message ?? "error"); - } else { - cb(undefined, data.response); - } - } - }); - }; - + /* + When the Syncdoc for Jupyter that is responsible for running computations + starts, it creates this service. + */ protected async api_call( endpoint: string, query?: any, timeout_ms?: number, ): Promise { - if (this._state === "closed" || this.syncdb == null) { + if (this._state === "closed") { throw Error("closed -- jupyter actions -- api_call"); } - if (this.syncdb.get_state() == "init") { - await once(this.syncdb, "ready"); - } - if (this.apiCallHandler == null) { - this.initApiCallHandler(); - } - if (this.apiCallHandler == null) { - throw Error("bug"); + if (this._client.callNatsService == null) { + throw Error("api not available"); } - - this.apiCallHandler.id += 1; - const { id, responseCallbacks } = this.apiCallHandler; - await this.syncdb.sendMessageToProject({ - event: "api-request", - id, + console.log("callNatsService ", { endpoint, query, timeout_ms }); + const resp = await this._client.callNatsService({ + project_id: this.project_id, path: this.path, - endpoint, - query, + service: "api", + mesg: { endpoint, query }, + timeout: timeout_ms, }); - const waitForResponse = (cb) => { - if (timeout_ms) { - setTimeout(() => { - if (responseCallbacks[id] == null) return; - cb("timeout"); - delete responseCallbacks[id]; - }, timeout_ms); - } - responseCallbacks[id] = cb; - }; - const resp = await callback(waitForResponse); + console.log("api_call got response", resp); return resp; } diff --git a/src/packages/jupyter/tsconfig.json b/src/packages/jupyter/tsconfig.json index fa655df897..56e63b5680 100644 --- a/src/packages/jupyter/tsconfig.json +++ b/src/packages/jupyter/tsconfig.json @@ -9,6 +9,7 @@ { "path": "../backend" }, { "path": "../sync" }, { "path": "../sync-client" }, - { "path": "../util" } + { "path": "../util" }, + { "path": "../nats" } ] } diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts index 40816204a7..f892bf2a40 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/nats/names.ts @@ -99,29 +99,28 @@ export function streamSubject({ } export function projectSubject({ - project_id, - compute_server_id = 0, - // service = optional name of the microservice, e.g., 'api', 'terminal' service, + project_id, + compute_server_id, // path = optional name of specific path for that microservice -- replaced by its sha1 path, }: { project_id: string; + service: string; compute_server_id?: number; - service?: string; path?: string; }): string { if (!project_id) { throw Error("project_id must be set"); } - let subject = `project.${project_id}.${compute_server_id}`; - if (service) { - subject += "." + service; - if (path) { - subject += "." + sha1(path); - } - } - return subject; + const segments = [ + "project", + project_id, + compute_server_id ?? "-", + service ?? "-", + path ? sha1(path) : "-", + ]; + return segments.join("."); } export function projectStreamName({ diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index d23e151915..9c965f33f5 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -16,9 +16,17 @@ "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "prepublishOnly": "pnpm test" }, - "files": ["dist/**", "README.md", "package.json"], + "files": [ + "dist/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["utilities", "nats", "cocalc"], + "keywords": [ + "utilities", + "nats", + "cocalc" + ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/comm": "workspace:*", @@ -26,6 +34,7 @@ "@cocalc/util": "workspace:*", "@nats-io/jetstream": "3.0.0-36", "@nats-io/kv": "3.0.0-30", + "@nats-io/services": "3.0.0-25", "awaiting": "^3.0.0", "events": "3.3.0", "immutable": "^4.3.0", diff --git a/src/packages/nats/service.ts b/src/packages/nats/service.ts new file mode 100644 index 0000000000..69f9e92a6a --- /dev/null +++ b/src/packages/nats/service.ts @@ -0,0 +1,138 @@ +/* +Simple to use UI to connect anything in cocalc via request/reply services. + +- callNatsService +- createNatsService + +The input is basically where the service is (account, project, public), +and either what message to send or how to handle messages. +Also if the handler throws an error, the caller will throw +an error too. +*/ + +import { Svcm } from "@nats-io/services"; +import { type NatsEnv } from "@cocalc/nats/types"; +import { sha1 } from "@cocalc/util/misc"; + +export interface ServiceDescription { + service: string; + project_id?: string; + account_id?: string; + compute_server_id?: number; + path?: string; +} + +export interface ServiceCall extends ServiceDescription { + mesg: any; + timeout?: number; + env?: NatsEnv; +} + +export async function callNatsService(opts: ServiceCall): Promise { + if (opts.env == null) { + throw Error("NATS env must be specified"); + } + const { nc, jc } = opts.env; + const subject = serviceSubject(opts); + const resp = await nc.request(subject, jc.encode(opts.mesg), { + timeout: opts.timeout, + }); + const result = jc.decode(resp.data); + if (result?.error) { + throw Error(result.error); + } + return result; +} + +export type CallNatsServiceFunction = typeof callNatsService; + +export async function createNatsService(options: Options) { + const s = new NatsService(options); + await s.init(); + return s; +} + +export type CreateNatsServiceFunction = typeof createNatsService; + +export function serviceSubject({ + service, + account_id, + project_id, + compute_server_id, + path, +}: ServiceDescription): string { + let segments; + if (!project_id && !account_id) { + segments = ["public", service]; + } else if (project_id) { + segments = [ + "services", + `project-${project_id}`, + compute_server_id ?? "-", + service, + path ? sha1(path) : "-", + ]; + } else if (account_id) { + segments = ["services", `account-${account_id}`, service]; + } + return segments.join("."); +} + +interface Options extends ServiceDescription { + env?: NatsEnv; + description?: string; + version?: string; + handler: (mesg) => Promise; +} + +export class NatsService { + private options: Options; + private subject: string; + private api?; + + constructor(options: Options) { + this.options = options; + this.subject = serviceSubject(options); + } + + init = async () => { + if (this.options.env == null) { + throw Error("NATS env must be specified"); + } + const svcm = new Svcm(this.options.env.nc); + + const service = await svcm.add({ + name: this.options.service, + version: this.options.version ?? "0.0.1", + description: this.options.description, + }); + + this.api = service.addEndpoint("api", { subject: this.subject }); + this.listen(); + }; + + private listen = async () => { + if (this.options.env == null) { + throw Error("NATS env must be specified"); + } + const jc = this.options.env.jc; + for await (const mesg of this.api) { + const request = jc.decode(mesg.data) ?? ({} as any); + let resp; + try { + resp = await this.options.handler(request); + } catch (err) { + resp = { error: `${err}` }; + } + mesg.respond(jc.encode(resp)); + } + }; + + close = () => { + this.api?.stop(); + // @ts-ignore + delete this.subject; + // @ts-ignore + delete this.options; + }; +} diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index bc235342d4..7aaa016bd9 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -972,6 +972,9 @@ importers: '@cocalc/jupyter': specifier: workspace:* version: 'link:' + '@cocalc/nats': + specifier: workspace:* + version: link:../nats '@cocalc/sync': specifier: workspace:* version: link:../sync @@ -1072,6 +1075,9 @@ importers: '@nats-io/kv': specifier: 3.0.0-30 version: 3.0.0-30 + '@nats-io/services': + specifier: 3.0.0-25 + version: 3.0.0-25 awaiting: specifier: ^3.0.0 version: 3.0.0 diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index c9b4f57b8b..0887a31e0f 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -51,6 +51,13 @@ import { get_syncdoc } from "./sync/sync-doc"; import synctable_nats from "@cocalc/project/nats/synctable"; import pubsub from "@cocalc/project/nats/pubsub"; import type { NatsSyncTableFunction } from "@cocalc/nats/sync/synctable"; +import { getEnv } from "@cocalc/project/nats/env"; +import { + callNatsService, + createNatsService, + type CallNatsServiceFunction, + type CreateNatsServiceFunction, +} from "@cocalc/nats/service"; const winston = getLogger("client"); @@ -516,6 +523,14 @@ export class Client extends EventEmitter implements ProjectClientInterface { return await pubsub({ path, name }); }; + callNatsService: CallNatsServiceFunction = async (options) => { + return await callNatsService({ ...options, env: await getEnv() }); + }; + + createNatsService: CreateNatsServiceFunction = async (options) => { + return createNatsService({ ...options, env: await getEnv() }); + }; + // WARNING: making two of the exact same sync_string or sync_db will definitely // lead to corruption! diff --git a/src/packages/project/nats/env.ts b/src/packages/project/nats/env.ts index 8aa9f5dfec..75eb7bceb9 100644 --- a/src/packages/project/nats/env.ts +++ b/src/packages/project/nats/env.ts @@ -2,8 +2,8 @@ import { sha1 } from "@cocalc/backend/sha1"; import getConnection from "./connection"; import { JSONCodec } from "nats"; +const jc = JSONCodec(); export async function getEnv() { const nc = await getConnection(); - const jc = JSONCodec(); return { sha1, nc, jc }; } diff --git a/src/packages/project/nats/names.ts b/src/packages/project/nats/names.ts index dcca25bdc8..9b1dbc1301 100644 --- a/src/packages/project/nats/names.ts +++ b/src/packages/project/nats/names.ts @@ -1,7 +1,7 @@ import { compute_server_id, project_id } from "@cocalc/project/data"; import { projectSubject, projectStreamName } from "@cocalc/nats/names"; -export function getSubject(opts: { path?; service? }) { +export function getSubject(opts: { path?: string; service: string }) { return projectSubject({ ...opts, compute_server_id, project_id }); } diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index a930dc51a0..ad0499d0b1 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -112,7 +112,6 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { const userType = getCoCalcUserType(cocalcUser); // TODO: jetstream permissions are WAY TO BROAD. const goalPub = new Set([ - // "_INBOX.>", // !!!! TODO: so can create responses to requests -- fix this it is horribly insecure!!!!! `hub.${userType}.${userId}.>`, // can talk as *only this user* to the hub's api's "$JS.API.INFO", ]); diff --git a/src/packages/sync/client/types.ts b/src/packages/sync/client/types.ts index 7437c85412..18e00b046e 100644 --- a/src/packages/sync/client/types.ts +++ b/src/packages/sync/client/types.ts @@ -1,5 +1,9 @@ import type { EventEmitter } from "events"; import type { CB } from "@cocalc/util/types/callback"; +import type { + CallNatsServiceFunction, + CreateNatsServiceFunction, +} from "@cocalc/nats/service"; // What we need the client to implement so we can use // it to support a table. @@ -17,6 +21,8 @@ export interface Client extends EventEmitter { touch_project: (project_id: string, compute_server_id?: number) => void; set_connected?: Function; is_deleted: (path: string, project_id: string) => true | false | undefined; + callNatsService?: CallNatsServiceFunction; + createNatsService?: CreateNatsServiceFunction; } export interface ClientFs extends Client { diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index eff1f3cd2e..4373711a50 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1303,6 +1303,16 @@ export class SyncDoc extends EventEmitter { atomic: false, immutable: true, }); + } else if (this.useNats) { + synctable = await this.client.synctable_nats(query, { + obj: { + project_id: this.project_id, + path: this.path, + }, + stream: false, + atomic: true, + immutable: true, + }); } else { switch (this.data_server) { case "project": diff --git a/src/packages/sync/editor/generic/types.ts b/src/packages/sync/editor/generic/types.ts index 44e3039d13..2c93f31595 100644 --- a/src/packages/sync/editor/generic/types.ts +++ b/src/packages/sync/editor/generic/types.ts @@ -12,6 +12,10 @@ import { SyncTable } from "@cocalc/sync/table/synctable"; import type { ExecuteCodeOptionsWithCallback } from "@cocalc/util/types/execute-code"; +import type { + CallNatsServiceFunction, + CreateNatsServiceFunction, +} from "@cocalc/nats/service"; export interface Patch { time: Date; // timestamp of when patch made @@ -97,6 +101,8 @@ export interface ProjectClient extends EventEmitter { synctable_nats: (query: any, obj?) => Promise; pubsub_nats: (query: any, obj?) => Promise; + callNatsService?: CallNatsServiceFunction; + createNatsService?: CreateNatsServiceFunction; // account_id or project_id or compute_server_id (encoded as a UUID - use decodeUUIDtoNum to decode) client_id: () => string; diff --git a/src/packages/sync/editor/string/test/client-test.ts b/src/packages/sync/editor/string/test/client-test.ts index 4fa52d41b1..0c4a03b54d 100644 --- a/src/packages/sync/editor/string/test/client-test.ts +++ b/src/packages/sync/editor/string/test/client-test.ts @@ -178,6 +178,10 @@ export class Client extends EventEmitter implements Client0 { throw Error("pubsub_nats: not implemented"); } + async natsRequest(_subject: string, _mesg: any, _options?) { + throw Error("natsRequest: not implemented"); + } + // account_id or project_id public client_id(): string { return this._client_id; From b8bdad4bc9b54802ccd9bc8204f93485760bfd65 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 14 Feb 2025 12:42:07 +0000 Subject: [PATCH 194/281] nats -- active jupyter --- src/packages/jupyter/redux/actions.ts | 78 ++++++++++--------- src/packages/jupyter/redux/project-actions.ts | 25 +++++- src/packages/nats/service.ts | 29 ++++++- src/packages/project/client.ts | 2 +- src/packages/project/nats/open-files.ts | 33 +++++++- src/packages/sync/editor/generic/sync-doc.ts | 4 +- 6 files changed, 125 insertions(+), 46 deletions(-) diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 18faba8ef9..0918f9d40f 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -40,6 +40,7 @@ import { import { SyncDB } from "@cocalc/sync/editor/db/sync"; import type { Client } from "@cocalc/sync/client/types"; import latexEnvs from "@cocalc/util/latex-envs"; +import { debounce } from "lodash"; const { close, required, defaults } = misc; @@ -204,7 +205,6 @@ export abstract class JupyterActions extends Actions { if (this._client.callNatsService == null) { throw Error("api not available"); } - console.log("callNatsService ", { endpoint, query, timeout_ms }); const resp = await this._client.callNatsService({ project_id: this.project_id, path: this.path, @@ -212,7 +212,6 @@ export abstract class JupyterActions extends Actions { mesg: { endpoint, query }, timeout: timeout_ms, }); - console.log("api_call got response", resp); return resp; } @@ -274,43 +273,50 @@ export abstract class JupyterActions extends Actions { // real version is in derived class that project runs. } - fetch_jupyter_kernels = async (): Promise => { - let data; - const f = async () => { - data = await this.api_call("kernels", undefined, 5000); + fetch_jupyter_kernels = debounce( + reuseInFlight(async (): Promise => { + let data; + const f = async () => { + data = await this.api_call("kernels", undefined, 5000); + if (this._state === "closed") { + return; + } + }; + try { + await retry_until_success({ + max_time: 1000 * 15, // up to 15 seconds + start_delay: 3000, + max_delay: 10000, + f, + desc: "jupyter:fetch_jupyter_kernels", + }); + } catch (err) { + this.set_error(err); + return; + } if (this._state === "closed") { return; } - }; - try { - await retry_until_success({ - max_time: 1000 * 15, // up to 15 seconds - start_delay: 500, - max_delay: 5000, - f, - desc: "jupyter:fetch_jupyter_kernels", - }); - } catch (err) { - this.set_error(err); - return; - } - if (this._state === "closed") { - return; - } - // we filter kernels that are disabled for the cocalc notebook – motivated by a broken GAP kernel - const kernels = immutable - .fromJS(data ?? []) - .filter((k) => !k.getIn(["metadata", "cocalc", "disabled"], false)); - const key: string = await this.store.jupyter_kernel_key(); - jupyter_kernels = jupyter_kernels.set(key, kernels); // global - this.setState({ kernels }); - // We must also update the kernel info (e.g., display name), now that we - // know the kernels (e.g., maybe it changed or is now known but wasn't before). - const kernel_info = this.store.get_kernel_info(this.store.get("kernel")); - this.setState({ kernel_info }); - await this.update_select_kernel_data(); // e.g. "kernel_selection" is drived from "kernels" - this.check_select_kernel(); - }; + // we filter kernels that are disabled for the cocalc notebook – motivated by a broken GAP kernel + const kernels = immutable + .fromJS(data ?? []) + .filter((k) => !k.getIn(["metadata", "cocalc", "disabled"], false)); + const key: string = await this.store.jupyter_kernel_key(); + jupyter_kernels = jupyter_kernels.set(key, kernels); // global + this.setState({ kernels }); + // We must also update the kernel info (e.g., display name), now that we + // know the kernels (e.g., maybe it changed or is now known but wasn't before). + const kernel_info = this.store.get_kernel_info(this.store.get("kernel")); + this.setState({ kernel_info }); + // e.g. "kernel_selection" is derived from "kernels" + await this.update_select_kernel_data(); + this.check_select_kernel(); + }), + // this debounce basically "caches the result" for this long + // after attempts to get the kernels: + 3000, + { leading: true, trailing: false }, + ); set_jupyter_kernels = async () => { if (this.store == null) return; diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 0da12764b8..2e3341a2c8 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -164,6 +164,9 @@ export class JupyterActions extends JupyterActions0 { dbg(); this._initialize_manager_already_done = true; + dbg("initialize Jupyter NATS api handler"); + await this.initNatsApi(); + this.sync_exec_state = debounce(this.sync_exec_state, 2000); this._throttled_ensure_positions_are_unique = debounce( this.ensure_positions_are_unique, @@ -202,9 +205,29 @@ export class JupyterActions extends JupyterActions0 { this.syncdb.on("cursor_activity", this.checkForComputeServerStateChange); // initialize the websocket api - this.initWebsocketApi(); + if (false) { + this.initWebsocketApi(); + } } + private initNatsApi = async () => { + if (this._client.createNatsService == null) { + throw Error("unable to initialize nats API"); + } + const api = await this._client.createNatsService({ + service: "api", + project_id: this.project_id, + path: this.path, + description: `Jupyter API for "${this.path}"`, + handler: async ({ endpoint, query }) => { + return await handleApiRequest(this.path, endpoint, query); + }, + }); + this.syncdb.on("closed", () => { + api.close(); + }); + }; + private async _first_load() { const dbg = this.dbg("_first_load"); dbg("doing load"); diff --git a/src/packages/nats/service.ts b/src/packages/nats/service.ts index 69f9e92a6a..e29c59492a 100644 --- a/src/packages/nats/service.ts +++ b/src/packages/nats/service.ts @@ -20,6 +20,7 @@ export interface ServiceDescription { account_id?: string; compute_server_id?: number; path?: string; + description?: string; } export interface ServiceCall extends ServiceDescription { @@ -78,6 +79,30 @@ export function serviceSubject({ return segments.join("."); } +export function serviceName({ + service, + account_id, + project_id, + compute_server_id, +}: ServiceDescription): string { + let segments; + if (!project_id && !account_id) { + segments = [service]; + } else if (project_id) { + segments = [`project-${project_id}`, compute_server_id ?? "-", service]; + } else if (account_id) { + segments = [`account-${account_id}`, service]; + } + return segments.join("-"); +} + +export function serviceDescription({ + description, + path, +}: ServiceDescription): string { + return [description, path ? `\nPath: ${path}` : ""].join(""); +} + interface Options extends ServiceDescription { env?: NatsEnv; description?: string; @@ -102,9 +127,9 @@ export class NatsService { const svcm = new Svcm(this.options.env.nc); const service = await svcm.add({ - name: this.options.service, + name: serviceName(this.options), version: this.options.version ?? "0.0.1", - description: this.options.description, + description: serviceDescription(this.options), }); this.api = service.addEndpoint("api", { subject: this.subject }); diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index 0887a31e0f..5be8618495 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -659,7 +659,7 @@ export class Client extends EventEmitter implements ProjectClientInterface { // Returns unknown if don't know // Returns false if definitely not. public is_deleted(filename: string, _project_id: string) { - return getListingsTable()?.isDeleted(filename); + return !!getListingsTable()?.isDeleted(filename); } public async set_deleted( diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 27d233cdd1..cf23e24f3d 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -43,6 +43,9 @@ import { SyncDB } from "@cocalc/sync/editor/db/sync"; import getLogger from "@cocalc/backend/logger"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { delay } from "awaiting"; +import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; +import { filename_extension, original_path } from "@cocalc/util/misc"; +import { get_blob_store } from "@cocalc/jupyter/blobs"; const logger = getLogger("project:nats:open-files"); @@ -188,22 +191,44 @@ const openSyncDoc = reuseInFlight(async (path: string) => { }); logger.debug("openSyncDoc got", { path, doctype }); - let doc; + let syncdoc; if (doctype.type == "string") { - doc = new SyncString({ + syncdoc = new SyncString({ ...doctype.opts, project_id, path, client, }); } else { - doc = new SyncDB({ + syncdoc = new SyncDB({ ...doctype.opts, project_id, path, client, }); } - openSyncDocs[path] = doc; + openSyncDocs[path] = syncdoc; + + syncdoc.on("error", (err) => { + closeSyncDoc(path); + openFiles?.setError(path, err); + logger.debug(`syncdoc error -- ${err}`, path); + }); + + // Extra backend support in some cases, e.g., Jupyter, Sage, etc. + const ext = filename_extension(path); + switch (ext) { + case "sage-jupyter2": + logger.debug("initializing Jupyter backend for ", path); + await get_blob_store(); // make sure jupyter blobstore is available + await initJupyterRedux(syncdoc, client); + const path1 = original_path(syncdoc.get_path()); + syncdoc.on("closed", async () => { + logger.debug("removing Jupyter backend for ", path1); + await removeJupyterRedux(path1, project_id); + }); + break; + } + return; }); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 4373711a50..fb9f61f25b 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -677,10 +677,10 @@ export class SyncDoc extends EventEmitter { this.on("user-change", this.throttled_file_use as any); } - private set_state(state: State): void { + private set_state = (state: State): void => { this.state = state; this.emit(state); - } + }; public get_state = (): State => { return this.state; From aae1da64e58dab9da07cfe025dbfb69505daf677 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 14 Feb 2025 14:32:45 +0000 Subject: [PATCH 195/281] nats: ipywidgets support --- src/packages/frontend/i18n/common.ts | 2 +- .../frontend/jupyter/codemirror-editor.tsx | 2 +- .../jupyter/output-messages/ipywidget.tsx | 62 ++++++++++++++++++- src/packages/jupyter/kernel/kernel.ts | 6 +- src/packages/jupyter/redux/project-actions.ts | 10 +-- src/packages/nats/sync/synctable-kv.ts | 5 +- .../sync/editor/generic/ipywidgets-state.ts | 7 ++- src/packages/sync/editor/generic/sync-doc.ts | 10 +++ src/packages/sync/table/changefeed-nats.ts | 4 +- src/packages/util/base64.ts | 5 +- 10 files changed, 92 insertions(+), 21 deletions(-) diff --git a/src/packages/frontend/i18n/common.ts b/src/packages/frontend/i18n/common.ts index 5a7386941b..7ddee2f86f 100644 --- a/src/packages/frontend/i18n/common.ts +++ b/src/packages/frontend/i18n/common.ts @@ -1180,7 +1180,7 @@ export const jupyter = { confirm_restart_body: { id: "jupyter.editor.confirm_restart.body", defaultMessage: - "Do you want to restart the kernel? All variable values will be lost.", + "Do you want to restart the kernel? All variable values will be lost. If you are restarted to detect a new package, restart *twice* due to the kernel pool.", }, confirm_halt_kernel_title: { id: "jupyter.editor.confirm_halt_kernel.title", diff --git a/src/packages/frontend/jupyter/codemirror-editor.tsx b/src/packages/frontend/jupyter/codemirror-editor.tsx index b3fb561ebb..0dc80b2b6c 100644 --- a/src/packages/frontend/jupyter/codemirror-editor.tsx +++ b/src/packages/frontend/jupyter/codemirror-editor.tsx @@ -812,7 +812,7 @@ export const CodeMirrorEditor: React.FC = ({ }} onClick={focus_cm} > -
+
Enter code{setShowAICellGen == null ? "..." : " or "}
(false); + const valueRef = useRef(value); + + // We have to wait a bit for ipywidgets_state.getSerializedModelState(id) + // to be defined, since state of the notebook and state of widgets are + // done in parallel, hence unpredicatable order, over the network + // (since using NATS instead of a single socket). + useEffect(() => { + (async () => { + const ipywidgets_state = actions?.widget_manager?.ipywidgets_state; + if (ipywidgets_state == null) { + setIsReady(true); + return; + } + const start = Date.now(); + const id = value.get("model_id"); + while (Date.now() - start <= MAX_WAIT) { + if (!valueRef.current.equals(value)) { + // let new function take over + return; + } + if (ipywidgets_state.getSerializedModelState(id) != null) { + /* + Without the delay, this fails the first time usually. I think + the delay just allows ipywidegts_state to process the batch of + messages that have arrived defining the state, rather than just + the first message: + +%matplotlib ipympl +import matplotlib.pyplot as plt +import numpy as np +fig, ax = plt.subplots() +x = np.linspace(0, 2*np.pi, 100) +y = np.sin(3*x) +ax.plot(x, y) + */ + await delay(1); + setIsReady(true); + return; + } else { + setIsReady(false); + } + try { + await once(ipywidgets_state, "change", MAX_WAIT); + } catch { + setIsReady(true); + return; + } + } + setIsReady(true); + })(); + }, [value]); useEffect(() => { - if (actions == null || !isVisible) { + if (actions == null || !isVisible || !isReady) { // console.log("IpyWidget: not rendering due to actions=null"); return; } @@ -76,10 +132,10 @@ export function IpyWidget({ id: cell_id, value, actions }: WidgetProps) { return () => { $(div).empty(); }; - }, [isVisible]); + }, [isVisible, isReady]); if (unknown) { - const msg = "Run cell to load widget."; + const msg = "Run cell to load widget"; return ( 0) { buffers = buffers64?.map((x) => Buffer.from(base64ToBuffer(x))) ?? []; dbg( @@ -1040,7 +1041,8 @@ class JupyterKernel extends EventEmitter implements JupyterKernelInterface { buffers, }; - dbg(message); + // HUGE + // dbg(message); // "The Kernel listens for these messages on the Shell channel, // and the Frontend listens for them on the IOPub channel." -- docs this.channel?.next(message); diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 2e3341a2c8..cf9085ea2b 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -199,7 +199,7 @@ export class JupyterActions extends JupyterActions0 { } this.syncdb.ipywidgets_state.on( "change", - this.handle_ipywidgets_state_change.bind(this), + this.handle_ipywidgets_state_change, ); this.syncdb.on("cursor_activity", this.checkForComputeServerStateChange); @@ -1379,7 +1379,7 @@ export class JupyterActions extends JupyterActions0 { // object changes, e.g., in response to a user moving a slider in the browser. // It crafts a comm message that is sent to the running Jupyter kernel telling // it about this change by calling send_comm_message_to_kernel. - private handle_ipywidgets_state_change(keys): void { + private handle_ipywidgets_state_change = (keys): void => { if (this.is_closed()) { return; } @@ -1441,10 +1441,12 @@ export class JupyterActions extends JupyterActions0 { ); */ } else { - throw Error(`invalid synctable state -- unknown type '${type}'`); + const m = `Jupyter: unknown type '${type}'`; + console.warn(m); + dbg(m); } } - } + }; public async process_comm_message_from_kernel(mesg: any): Promise { const dbg = this.dbg("process_comm_message_from_kernel"); diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index b023b1dc76..8306aee96f 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -110,7 +110,10 @@ export class SyncTableKV extends EventEmitter { x = { ...x, value: this.dkv?.get(x.key) }; } } - this.emit("change", x); + // change api was to emit array of keys. + // We also use this packages/sync/table/changefeed-nats.ts which needs the value, + // so we emit that object second. + this.emit("change", [x.key], x); }); this.set_state("connected"); }; diff --git a/src/packages/sync/editor/generic/ipywidgets-state.ts b/src/packages/sync/editor/generic/ipywidgets-state.ts index ed08e01e8a..789690c956 100644 --- a/src/packages/sync/editor/generic/ipywidgets-state.ts +++ b/src/packages/sync/editor/generic/ipywidgets-state.ts @@ -421,7 +421,7 @@ export class IpywidgetsState extends EventEmitter { fire_change_event: boolean = true, merge?: "none" | "shallow" | "deep", ): void => { - const dbg = this.dbg("set"); + //const dbg = this.dbg("set"); const string_id = this.syncdoc.get_string_id(); if (typeof data != "object") { throw Error("TypeError -- data must be a map"); @@ -431,7 +431,8 @@ export class IpywidgetsState extends EventEmitter { //defaultMerge = "shallow"; // we manually do the shallow merge only on the data field. const current = this.get_model_value(model_id); - dbg("value: before", { data, current }); + // this can be HUGE: + // dbg("value: before", { data, current }); if (current != null) { for (const k in data) { if (is_object(data[k]) && is_object(current[k])) { @@ -442,7 +443,7 @@ export class IpywidgetsState extends EventEmitter { } data = current; } - dbg("value -- after", { merged: data }); + // dbg("value -- after", { merged: data }); defaultMerge = "none"; } else if (type == "buffers") { // it's critical to not throw away existing buffers when diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index fb9f61f25b..32deda968b 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1303,6 +1303,16 @@ export class SyncDoc extends EventEmitter { atomic: false, immutable: true, }); + } else if (this.useNats && query.ipywidgets) { + synctable = await this.client.synctable_nats(query, { + obj: { + project_id: this.project_id, + path: this.path, + }, + stream: false, + atomic: true, + immutable: true, + }); } else if (this.useNats) { synctable = await this.client.synctable_nats(query, { obj: { diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index b2c7d09e49..9a94abe799 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -77,8 +77,8 @@ export class NatsChangefeed extends EventEmitter { return; } this.natsSynctable.on( - "change", - ({ key, value: new_val, prev: old_val }) => { + "changefeed", + (_, { key, value: new_val, prev: old_val }) => { let x; if (new_val == null) { x = { action: "delete", old_val, key }; diff --git a/src/packages/util/base64.ts b/src/packages/util/base64.ts index 15a7140fed..44e263e101 100644 --- a/src/packages/util/base64.ts +++ b/src/packages/util/base64.ts @@ -25,9 +25,6 @@ export function bufferToBase64(buffer: ArrayBuffer | ArrayBufferView): string { // Convert a base64 string to an ArrayBuffer. export function base64ToBuffer(base64: string): ArrayBuffer { const buffer = toUint8Array(base64).buffer; - // Typescript 5.7 related: https://devblogs.microsoft.com/typescript/announcing-typescript-5-7/ - if (buffer instanceof SharedArrayBuffer) { - throw new Error("SharedArrayBuffer is not supported"); - } + // @ts-ignore return buffer; } From bbed478bfd002d0ecd5247122957482be5b95c57 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 14 Feb 2025 15:18:16 +0000 Subject: [PATCH 196/281] nats: wire up code formatter --- .../frontend/project/websocket/api.ts | 9 ++++--- src/packages/nats/project-api/editor.ts | 14 ++-------- src/packages/project/client.ts | 6 ++++- src/packages/project/formatters/index.ts | 6 ++++- src/packages/project/nats/api/editor.ts | 4 --- src/packages/project/nats/formatter.ts | 26 +++++++++++++++++++ src/packages/project/nats/open-files.ts | 11 ++++++-- 7 files changed, 53 insertions(+), 23 deletions(-) create mode 100644 src/packages/project/nats/formatter.ts diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 668330068d..de8472acab 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -246,11 +246,14 @@ export class API { formatter = async ( path: string, config: FormatterConfig, - compute_server_id?: number, + //compute_server_id?: number, ): Promise => { const options: FormatterOptions = this.check_formatter_available(config); - const api = this.getApi({ compute_server_id }); - const { result } = await api.editor.formatter({ path, options }); + const { result } = await webapp_client.callNatsService({ + project_id: this.project_id, + service: "formatter", + mesg: { path, options }, + }); return result; }; diff --git a/src/packages/nats/project-api/editor.ts b/src/packages/nats/project-api/editor.ts index 2389c4ba57..89e9e876f2 100644 --- a/src/packages/nats/project-api/editor.ts +++ b/src/packages/nats/project-api/editor.ts @@ -1,16 +1,11 @@ import type { NbconvertParams } from "@cocalc/util/jupyter/types"; import type { RunNotebookOptions } from "@cocalc/util/jupyter/nbgrader-types"; -import type { - Options as FormatterOptions, - FormatResult, -} from "@cocalc/util/code-formatter"; +import type { Options as FormatterOptions } from "@cocalc/util/code-formatter"; export const editor = { jupyterStripNotebook: true, jupyterNbconvert: true, jupyterRunNotebook: true, - - formatter: true, formatterString: true, }; @@ -19,12 +14,7 @@ export interface Editor { jupyterNbconvert: (opts: NbconvertParams) => Promise; jupyterRunNotebook: (opts: RunNotebookOptions) => Promise; - // returns a patch to transform doc into formatted form. - formatter: (opts: { - path: string; - options: FormatterOptions; - }) => Promise<{ result: FormatResult }>; - + // returns a patch to transform str into formatted form. formatterString: (opts: { str: string; options: FormatterOptions; diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index 5be8618495..8167e0ad26 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -528,7 +528,11 @@ export class Client extends EventEmitter implements ProjectClientInterface { }; createNatsService: CreateNatsServiceFunction = async (options) => { - return createNatsService({ ...options, env: await getEnv() }); + return createNatsService({ + ...options, + project_id: this.project_id, + env: await getEnv(), + }); }; // WARNING: making two of the exact same sync_string or sync_db will definitely diff --git a/src/packages/project/formatters/index.ts b/src/packages/project/formatters/index.ts index c9531e7950..8e50eb38f3 100644 --- a/src/packages/project/formatters/index.ts +++ b/src/packages/project/formatters/index.ts @@ -46,13 +46,17 @@ const logger = getLogger("project:formatters"); export async function run_formatter({ path, options, + syncstring, }: { path: string; options: Options; + syncstring?; }): Promise { const client = getClient(); // What we do is edit the syncstring with the given path to be "prettier" if possible... - const syncstring = client.syncdoc({ path }); + if (syncstring == null) { + syncstring = client.syncdoc({ path }); + } if (syncstring == null || syncstring.get_state() == "closed") { return { status: "error", diff --git a/src/packages/project/nats/api/editor.ts b/src/packages/project/nats/api/editor.ts index c79bf3939e..e3f7cf688c 100644 --- a/src/packages/project/nats/api/editor.ts +++ b/src/packages/project/nats/api/editor.ts @@ -3,7 +3,3 @@ export { jupyter_run_notebook as jupyterRunNotebook } from "@cocalc/jupyter/nbgr export { nbconvert as jupyterNbconvert } from "../../jupyter/convert"; export { run_formatter_string as formatterString } from "../../formatters"; -import { run_formatter } from "../../formatters"; -export async function formatter(opts) { - return { result: await run_formatter(opts) }; -} diff --git a/src/packages/project/nats/formatter.ts b/src/packages/project/nats/formatter.ts new file mode 100644 index 0000000000..76504e362c --- /dev/null +++ b/src/packages/project/nats/formatter.ts @@ -0,0 +1,26 @@ +/* +File formatting service. +*/ + +import { getClient } from "@cocalc/project/client"; +import { run_formatter, type Options } from "../formatters"; + +interface Message { + path: string; + options: Options; +} + +export async function createFormatterService({ openSyncDocs }) { + const client = getClient(); + return await client.createNatsService({ + service: "formatter", + description: "Format code in an open file.", + handler: async (opts: Message) => { + const syncstring = openSyncDocs[opts.path]; + if (syncstring == null) { + throw Error(`"${opts.path}" is not opened`); + } + return { result: await run_formatter({ ...opts, syncstring }) }; + }, + }); +} diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index cf23e24f3d..22dadf79b8 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -5,7 +5,7 @@ DEVELOPMENT: 0. From the browser, terminate open-files api service running in the project already, if any - await cc.client.nats_client.projectApi({project_id:'81e0c408-ac65-4114-bad5-5f4b6539bd0e'}).system.terminate({service:'open-files'}) + await cc.client.nats_client.projectApi({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'}).system.terminate({service:'open-files'}) // {status: 'terminated', service: 'open-files'} @@ -46,10 +46,12 @@ import { delay } from "awaiting"; import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; import { filename_extension, original_path } from "@cocalc/util/misc"; import { get_blob_store } from "@cocalc/jupyter/blobs"; +import { createFormatterService } from "./formatter"; const logger = getLogger("project:nats:open-files"); let openFiles: OpenFiles | null = null; +let formatter: any = null; export async function init() { logger.debug("init"); @@ -68,8 +70,10 @@ export async function init() { handleChange(entry); }); + formatter = await createFormatterService({ openSyncDocs }); + // usefule for development - return { openFiles, openSyncDocs }; + return { openFiles, openSyncDocs, formatter, terminate }; } export function terminate() { @@ -79,6 +83,9 @@ export function terminate() { } openFiles?.close(); openFiles = null; + + formatter?.close(); + formatter = null; } const openSyncDocs: { [path: string]: SyncDoc } = {}; From e6b638fbc6f9dbac188052cfa68d7cfb22ebe868 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 14 Feb 2025 17:17:38 +0000 Subject: [PATCH 197/281] nats: better call error messages --- src/packages/nats/service.ts | 28 +++++++++++++++++--- src/packages/project/nats/open-files.ts | 2 +- src/packages/sync/editor/generic/sync-doc.ts | 6 ++++- src/packages/util/async-utils.ts | 13 +-------- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/packages/nats/service.ts b/src/packages/nats/service.ts index e29c59492a..8ea72ef12f 100644 --- a/src/packages/nats/service.ts +++ b/src/packages/nats/service.ts @@ -12,7 +12,9 @@ an error too. import { Svcm } from "@nats-io/services"; import { type NatsEnv } from "@cocalc/nats/types"; -import { sha1 } from "@cocalc/util/misc"; +import { sha1, trunc_middle } from "@cocalc/util/misc"; + +const DEFAULT_TIMEOUT = 5000; export interface ServiceDescription { service: string; @@ -35,9 +37,27 @@ export async function callNatsService(opts: ServiceCall): Promise { } const { nc, jc } = opts.env; const subject = serviceSubject(opts); - const resp = await nc.request(subject, jc.encode(opts.mesg), { - timeout: opts.timeout, - }); + let resp; + const timeout = opts.timeout ?? DEFAULT_TIMEOUT; + try { + resp = await nc.request(subject, jc.encode(opts.mesg), { + timeout, + }); + } catch (err) { + if (err.name == "NatsError") { + const p = opts.path ? `${trunc_middle(opts.path, 64)}:` : ""; + if (err.code == "503") { + throw Error( + `Not Available: service ${p}${opts.service} is not available`, + ); + } else if (err.code == "TIMEOUT") { + throw Error( + `Timeout: service ${p}${opts.service} did not respond for ${Math.round(timeout / 1000)} seconds`, + ); + } + } + throw err; + } const result = jc.decode(resp.data); if (result?.error) { throw Error(result.error); diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 22dadf79b8..1ead1332bd 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -140,7 +140,7 @@ function supportAutoclose(path: string): boolean { async function closeIgnoredFilesLoop() { while (openFiles != null && openFiles.state == "connected") { await delay(NATS_OPEN_FILE_TOUCH_INTERVAL); - if (openFiles.state != "connected") { + if (openFiles?.state != "connected") { return; } const paths = Object.keys(openSyncDocs); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 32deda968b..44fe95675f 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1716,7 +1716,11 @@ export class SyncDoc extends EventEmitter { }; private update_if_file_is_read_only = async (): Promise => { - this.set_read_only(await this.file_is_read_only()); + const read_only = await this.file_is_read_only(); + if (this.state == "closed") { + return; + } + this.set_read_only(read_only); }; private init_load_from_disk = async (): Promise => { diff --git a/src/packages/util/async-utils.ts b/src/packages/util/async-utils.ts index f25f4a7b6c..2fffa34cc6 100644 --- a/src/packages/util/async-utils.ts +++ b/src/packages/util/async-utils.ts @@ -39,17 +39,6 @@ export function callback_opts(f: Function) { }; } -/** - * convert the given error to a string, by either serializing the object or returning the string as it is - */ -function err2str(err: any): string { - if (typeof err === "string") { - return err; - } else { - return JSON.stringify(err); - } -} - /* retry_until_success keeps calling an async function f with exponential backoff until f does NOT raise an exception. Then retry_until_success returns whatever f returned. @@ -110,7 +99,7 @@ export async function retry_until_success( let e; if (last_exc) { e = Error( - `${err} -- last error was ${err2str(last_exc)} -- ${opts.desc}`, + `${err} -- last error was '${last_exc}' -- ${opts.desc}`, ); } else { e = Error(`${err} -- ${opts.desc}`); From eb0d1b4de0abc7d64ea55122a665149d5b0d8953 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 14 Feb 2025 19:51:22 +0000 Subject: [PATCH 198/281] nats service: add a typed option --- src/packages/backend/nats/index.ts | 9 ++- src/packages/frontend/client/client.ts | 6 ++ .../frontend/project/websocket/api.ts | 10 ++-- src/packages/nats/client.ts | 17 ++++++ src/packages/nats/package.json | 13 +--- src/packages/nats/service/index.ts | 7 +++ src/packages/nats/service/project.ts | 16 +++++ src/packages/nats/{ => service}/service.ts | 4 +- src/packages/nats/service/typed.ts | 60 +++++++++++++++++++ src/packages/nats/types.ts | 2 + src/packages/project/client.ts | 11 +++- src/packages/project/nats/formatter.ts | 22 ++++--- src/packages/project/nats/open-files.ts | 4 ++ 13 files changed, 149 insertions(+), 32 deletions(-) create mode 100644 src/packages/nats/client.ts create mode 100644 src/packages/nats/service/index.ts create mode 100644 src/packages/nats/service/project.ts rename src/packages/nats/{ => service}/service.ts (96%) create mode 100644 src/packages/nats/service/typed.ts diff --git a/src/packages/backend/nats/index.ts b/src/packages/backend/nats/index.ts index 9a78b013c1..eca8361505 100644 --- a/src/packages/backend/nats/index.ts +++ b/src/packages/backend/nats/index.ts @@ -3,11 +3,18 @@ import { nats } from "@cocalc/backend/data"; import { readFile } from "node:fs/promises"; import getLogger from "@cocalc/backend/logger"; import { connect, credsAuthenticator } from "nats"; -export { getEnv } from "./env"; +import { getEnv } from "./env"; +export { getEnv }; import { delay } from "awaiting"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { CONNECT_OPTIONS } from "@cocalc/util/nats"; import { inboxPrefix } from "@cocalc/nats/names"; +import { setNatsClient } from "@cocalc/nats/client"; + +export function init() { + setNatsClient({ getNatsEnv: getEnv }); +} +init(); const logger = getLogger("backend:nats"); diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index f0b6008c77..7aa29791dd 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -37,6 +37,8 @@ import type { CallNatsServiceFunction, CreateNatsServiceFunction, } from "@cocalc/nats/service"; +import type { NatsEnvFunction } from "@cocalc/nats/types"; +import { setNatsClient } from "@cocalc/nats/client"; // This DEBUG variable comes from webpack: declare const DEBUG; @@ -91,6 +93,7 @@ export interface WebappClient extends EventEmitter { synctable_nats: NatsSyncTableFunction; callNatsService: CallNatsServiceFunction; createNatsService: CreateNatsServiceFunction; + getNatsEnv: NatsEnvFunction; pubsub_nats: Function; project_websocket: Function; prettier: Function; @@ -175,6 +178,7 @@ class Client extends EventEmitter implements WebappClient { synctable_nats: NatsSyncTableFunction; callNatsService: CallNatsServiceFunction; createNatsService: CreateNatsServiceFunction; + getNatsEnv: NatsEnvFunction; pubsub_nats: Function; project_websocket: Function; prettier: Function; @@ -272,6 +276,7 @@ class Client extends EventEmitter implements WebappClient { this.pubsub_nats = this.nats_client.pubsub; this.callNatsService = this.nats_client.callNatsService; this.createNatsService = this.nats_client.createNatsService; + this.getNatsEnv = this.nats_client.getEnv; this.query = this.query_client.query.bind(this.query_client); this.async_query = this.query_client.query.bind(this.query_client); @@ -371,3 +376,4 @@ class Client extends EventEmitter implements WebappClient { } export const webapp_client = new Client(); +setNatsClient(webapp_client); diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index de8472acab..6aa8ac3d38 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -36,6 +36,7 @@ import type { ExecuteCodeOutput, ExecuteCodeOptions, } from "@cocalc/util/types/execute-code"; +import * as services from "@cocalc/nats/service/project"; export class API { private conn; @@ -246,15 +247,14 @@ export class API { formatter = async ( path: string, config: FormatterConfig, - //compute_server_id?: number, + compute_server_id?: number, ): Promise => { const options: FormatterOptions = this.check_formatter_available(config); - const { result } = await webapp_client.callNatsService({ + const formatter = services.formatter({ project_id: this.project_id, - service: "formatter", - mesg: { path, options }, + compute_server_id, }); - return result; + return await formatter({ path, options }); }; formatter_string = async ( diff --git a/src/packages/nats/client.ts b/src/packages/nats/client.ts new file mode 100644 index 0000000000..4ae5e91fc7 --- /dev/null +++ b/src/packages/nats/client.ts @@ -0,0 +1,17 @@ +import type { NatsEnv, NatsEnvFunction } from "@cocalc/nats/types"; + +interface Client { + getNatsEnv: NatsEnvFunction; +} + +let globalClient: null | Client = null; +export function setNatsClient(client: Client) { + globalClient = client; +} + +export async function getEnv(): Promise { + if (globalClient == null) { + throw Error("must set the global NATS client"); + } + return await globalClient.getNatsEnv(); +} diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 9c965f33f5..7f0519fb27 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -6,6 +6,7 @@ "./sync/*": "./dist/sync/*.js", "./hub-api": "./dist/hub-api/index.js", "./hub-api/*": "./dist/hub-api/*.js", + "./service": "./dist/service/index.js", "./project-api": "./dist/project-api/index.js", "./browser-api": "./dist/browser-api/index.js", "./*": "./dist/*.js" @@ -16,17 +17,9 @@ "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "prepublishOnly": "pnpm test" }, - "files": [ - "dist/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "nats", - "cocalc" - ], + "keywords": ["utilities", "nats", "cocalc"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/comm": "workspace:*", diff --git a/src/packages/nats/service/index.ts b/src/packages/nats/service/index.ts new file mode 100644 index 0000000000..6aa028b588 --- /dev/null +++ b/src/packages/nats/service/index.ts @@ -0,0 +1,7 @@ +export type { + ServiceDescription, + CallNatsServiceFunction, + ServiceCall, + CreateNatsServiceFunction, +} from "./service"; +export { callNatsService, createNatsService } from "./service"; diff --git a/src/packages/nats/service/project.ts b/src/packages/nats/service/project.ts new file mode 100644 index 0000000000..485b008162 --- /dev/null +++ b/src/packages/nats/service/project.ts @@ -0,0 +1,16 @@ +/* +Services in a project. +*/ + +import { natsService } from "./typed"; + +import type { + Options as FormatterOptions, + FormatResult, +} from "@cocalc/util/code-formatter"; + +export function formatter({ compute_server_id = 0, project_id }) { + return natsService<{ path: string; options: FormatterOptions }, FormatResult>( + { project_id, compute_server_id, service: "formatter" }, + ); +} diff --git a/src/packages/nats/service.ts b/src/packages/nats/service/service.ts similarity index 96% rename from src/packages/nats/service.ts rename to src/packages/nats/service/service.ts index 8ea72ef12f..e368bae093 100644 --- a/src/packages/nats/service.ts +++ b/src/packages/nats/service/service.ts @@ -32,6 +32,7 @@ export interface ServiceCall extends ServiceDescription { } export async function callNatsService(opts: ServiceCall): Promise { + // console.log("callNatsService", opts); if (opts.env == null) { throw Error("NATS env must be specified"); } @@ -123,7 +124,7 @@ export function serviceDescription({ return [description, path ? `\nPath: ${path}` : ""].join(""); } -interface Options extends ServiceDescription { +export interface Options extends ServiceDescription { env?: NatsEnv; description?: string; version?: string; @@ -163,6 +164,7 @@ export class NatsService { const jc = this.options.env.jc; for await (const mesg of this.api) { const request = jc.decode(mesg.data) ?? ({} as any); + // console.log("handle nats service call", request); let resp; try { resp = await this.options.handler(request); diff --git a/src/packages/nats/service/typed.ts b/src/packages/nats/service/typed.ts new file mode 100644 index 0000000000..6a650967da --- /dev/null +++ b/src/packages/nats/service/typed.ts @@ -0,0 +1,60 @@ +import { callNatsService, createNatsService } from "./service"; +import type { NatsService as NatsService0, Options } from "./service"; +import { getEnv } from "@cocalc/nats/client"; + +export function natsService( + options: Omit, +) { + const S = new NatsService(options); + return S as CallableNatsServiceInstance; +} + +interface CallableNatsService { + (mesg: Message, timeout?: number): Promise; +} + +export type CallableNatsServiceInstance = NatsService< + Message, + Response +> & + CallableNatsService; + +export class NatsService { + private service?: NatsService0; + private options: Omit; + + constructor(options: Omit) { + this.options = options; + return new Proxy(this, { + apply: (target, _thisArg, argumentsList) => { + return target.call.apply(target, argumentsList); + }, + }); + } + + listen = async (handler: (mesg: Message) => Promise) => { + this.service = await createNatsService({ + ...this.options, + handler, + env: await getEnv(), + }); + return this.service; + }; + + close = () => { + this.service?.close(); + delete this.service; + // @ts-ignore + delete this.options; + }; + + call = async (mesg: Message, timeout?: number): Promise => { + const resp = await callNatsService({ + ...this.options, + env: await getEnv(), + timeout, + mesg, + }); + return resp as Response; + }; +} diff --git a/src/packages/nats/types.ts b/src/packages/nats/types.ts index c09afecb2a..375848918b 100644 --- a/src/packages/nats/types.ts +++ b/src/packages/nats/types.ts @@ -6,3 +6,5 @@ export interface NatsEnv { } export type State = "disconnected" | "connected" | "closed"; + +export type NatsEnvFunction = () => Promise; diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index 8167e0ad26..f5a415979a 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -51,13 +51,15 @@ import { get_syncdoc } from "./sync/sync-doc"; import synctable_nats from "@cocalc/project/nats/synctable"; import pubsub from "@cocalc/project/nats/pubsub"; import type { NatsSyncTableFunction } from "@cocalc/nats/sync/synctable"; -import { getEnv } from "@cocalc/project/nats/env"; +import { getEnv as getNatsEnv } from "@cocalc/project/nats/env"; import { callNatsService, createNatsService, type CallNatsServiceFunction, type CreateNatsServiceFunction, } from "@cocalc/nats/service"; +import type { NatsEnvFunction } from "@cocalc/nats/types"; +import { setNatsClient } from "@cocalc/nats/client"; const winston = getLogger("client"); @@ -91,6 +93,7 @@ export function init() { throw Error("BUG: Client already initialized!"); } client = new Client(); + setNatsClient(client); return client; } @@ -524,17 +527,19 @@ export class Client extends EventEmitter implements ProjectClientInterface { }; callNatsService: CallNatsServiceFunction = async (options) => { - return await callNatsService({ ...options, env: await getEnv() }); + return await callNatsService({ ...options, env: await getNatsEnv() }); }; createNatsService: CreateNatsServiceFunction = async (options) => { return createNatsService({ ...options, project_id: this.project_id, - env: await getEnv(), + env: await getNatsEnv(), }); }; + getNatsEnv: NatsEnvFunction = async () => await getNatsEnv(); + // WARNING: making two of the exact same sync_string or sync_db will definitely // lead to corruption! diff --git a/src/packages/project/nats/formatter.ts b/src/packages/project/nats/formatter.ts index 76504e362c..196d491f81 100644 --- a/src/packages/project/nats/formatter.ts +++ b/src/packages/project/nats/formatter.ts @@ -2,25 +2,23 @@ File formatting service. */ -import { getClient } from "@cocalc/project/client"; import { run_formatter, type Options } from "../formatters"; +import * as services from "@cocalc/nats/service/project"; +import { compute_server_id, project_id } from "@cocalc/project/data"; interface Message { path: string; options: Options; } +const formatter = services.formatter({ compute_server_id, project_id }); + export async function createFormatterService({ openSyncDocs }) { - const client = getClient(); - return await client.createNatsService({ - service: "formatter", - description: "Format code in an open file.", - handler: async (opts: Message) => { - const syncstring = openSyncDocs[opts.path]; - if (syncstring == null) { - throw Error(`"${opts.path}" is not opened`); - } - return { result: await run_formatter({ ...opts, syncstring }) }; - }, + return formatter.listen(async (opts: Message) => { + const syncstring = openSyncDocs[opts.path]; + if (syncstring == null) { + throw Error(`"${opts.path}" is not opened`); + } + return await run_formatter({ ...opts, syncstring }); }); } diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 1ead1332bd..9b8b341de0 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -48,6 +48,9 @@ import { filename_extension, original_path } from "@cocalc/util/misc"; import { get_blob_store } from "@cocalc/jupyter/blobs"; import { createFormatterService } from "./formatter"; +// ensure nats connection stuff is initialized +import "@cocalc/backend/nats"; + const logger = getLogger("project:nats:open-files"); let openFiles: OpenFiles | null = null; @@ -55,6 +58,7 @@ let formatter: any = null; export async function init() { logger.debug("init"); + openFiles = await createOpenFiles(); // initialize From 12957dd25731dfee4bab1d9ff656668141979850 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 14 Feb 2025 20:06:24 +0000 Subject: [PATCH 199/281] switch jupyter to use new typed api - but no actual interesting typing yet --- .../frontend/project/websocket/api.ts | 2 +- src/packages/jupyter/redux/actions.ts | 11 +++-------- src/packages/jupyter/redux/project-actions.ts | 16 ++++++++-------- src/packages/nats/service/project.ts | 19 +++++++++++++++++++ src/packages/nats/service/typed.ts | 18 +----------------- 5 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 6aa8ac3d38..e9b6f87a3a 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -254,7 +254,7 @@ export class API { project_id: this.project_id, compute_server_id, }); - return await formatter({ path, options }); + return await formatter.call({ path, options }); }; formatter_string = async ( diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 0918f9d40f..c709605852 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -41,6 +41,7 @@ import { SyncDB } from "@cocalc/sync/editor/db/sync"; import type { Client } from "@cocalc/sync/client/types"; import latexEnvs from "@cocalc/util/latex-envs"; import { debounce } from "lodash"; +import { jupyter as natsJupyterService } from "@cocalc/nats/service/project"; const { close, required, defaults } = misc; @@ -202,17 +203,11 @@ export abstract class JupyterActions extends Actions { if (this._state === "closed") { throw Error("closed -- jupyter actions -- api_call"); } - if (this._client.callNatsService == null) { - throw Error("api not available"); - } - const resp = await this._client.callNatsService({ + const service = natsJupyterService({ project_id: this.project_id, path: this.path, - service: "api", - mesg: { endpoint, query }, - timeout: timeout_ms, }); - return resp; + return await service.call({ endpoint, query }, timeout_ms); } protected dbg = (f: string) => { diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index cf9085ea2b..4b04f81df3 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -34,6 +34,7 @@ import { handleApiRequest } from "@cocalc/jupyter/kernel/websocket-api"; import { callback } from "awaiting"; import { get_blob_store } from "@cocalc/jupyter/blobs"; import { removeJupyterRedux } from "@cocalc/jupyter/kernel"; +import { jupyter as natsJupyterService } from "@cocalc/nats/service/project"; // see https://github.com/sagemathinc/cocalc/issues/8060 const MAX_OUTPUT_SAVE_DELAY = 30000; @@ -214,17 +215,16 @@ export class JupyterActions extends JupyterActions0 { if (this._client.createNatsService == null) { throw Error("unable to initialize nats API"); } - const api = await this._client.createNatsService({ - service: "api", + const path = this.path; + const service = natsJupyterService({ project_id: this.project_id, - path: this.path, - description: `Jupyter API for "${this.path}"`, - handler: async ({ endpoint, query }) => { - return await handleApiRequest(this.path, endpoint, query); - }, + path, + }); + service.listen(async ({ endpoint, query }) => { + return await handleApiRequest(path, endpoint, query); }); this.syncdb.on("closed", () => { - api.close(); + service.close(); }); }; diff --git a/src/packages/nats/service/project.ts b/src/packages/nats/service/project.ts index 485b008162..55585dc666 100644 --- a/src/packages/nats/service/project.ts +++ b/src/packages/nats/service/project.ts @@ -9,8 +9,27 @@ import type { FormatResult, } from "@cocalc/util/code-formatter"; +// TODO: we may change it to NOT take compute server and have this listening from +// project and all compute servers... and have only the one with the file open +// actually reply. export function formatter({ compute_server_id = 0, project_id }) { return natsService<{ path: string; options: FormatterOptions }, FormatResult>( { project_id, compute_server_id, service: "formatter" }, ); } + +interface JupyterApiMessage { + endpoint: string; + query?: any; +} + +type JupyterApiResponse = any; + +export function jupyter({ project_id, path }) { + return natsService({ + project_id, + path, + service: "api", + description: "Jupyter notebook compute API", + }); +} diff --git a/src/packages/nats/service/typed.ts b/src/packages/nats/service/typed.ts index 6a650967da..0bcc387673 100644 --- a/src/packages/nats/service/typed.ts +++ b/src/packages/nats/service/typed.ts @@ -5,31 +5,15 @@ import { getEnv } from "@cocalc/nats/client"; export function natsService( options: Omit, ) { - const S = new NatsService(options); - return S as CallableNatsServiceInstance; + return new NatsService(options); } -interface CallableNatsService { - (mesg: Message, timeout?: number): Promise; -} - -export type CallableNatsServiceInstance = NatsService< - Message, - Response -> & - CallableNatsService; - export class NatsService { private service?: NatsService0; private options: Omit; constructor(options: Omit) { this.options = options; - return new Proxy(this, { - apply: (target, _thisArg, argumentsList) => { - return target.call.apply(target, argumentsList); - }, - }); } listen = async (handler: (mesg: Message) => Promise) => { From 0e32a105fe61e05abf44e685da02e037508619ba Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 14 Feb 2025 20:14:33 +0000 Subject: [PATCH 200/281] jupyter api: slightly better typing --- src/packages/jupyter/kernel/websocket-api.ts | 10 +++++----- src/packages/jupyter/redux/actions.ts | 5 +++-- src/packages/nats/service/project.ts | 14 +++++++++++++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/packages/jupyter/kernel/websocket-api.ts b/src/packages/jupyter/kernel/websocket-api.ts index a9eeeae013..26d8461ad3 100644 --- a/src/packages/jupyter/kernel/websocket-api.ts +++ b/src/packages/jupyter/kernel/websocket-api.ts @@ -11,18 +11,18 @@ various messages related to working with Jupyter. import { get_existing_kernel } from "@cocalc/jupyter/kernel"; import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data"; import { bufferToBase64 } from "@cocalc/util/base64"; +import { type JupyterApiEndpoint } from "@cocalc/nats/service/project"; export async function handleApiRequest( path: string, - endpoint: string, + endpoint: JupyterApiEndpoint, query?: any, ): Promise { // First handle endpoints that do not depend on a specific kernel. - switch (endpoint) { - case "kernels": - return await get_kernel_data(); + if(endpoint == 'kernels') { + return await get_kernel_data(); } - + // Now endpoints that do depend on a specific kernel. const kernel = get_existing_kernel(path); if (kernel == null) { diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index c709605852..6c61fd52a5 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -42,6 +42,7 @@ import type { Client } from "@cocalc/sync/client/types"; import latexEnvs from "@cocalc/util/latex-envs"; import { debounce } from "lodash"; import { jupyter as natsJupyterService } from "@cocalc/nats/service/project"; +import { type JupyterApiEndpoint } from "@cocalc/nats/service/project"; const { close, required, defaults } = misc; @@ -196,10 +197,10 @@ export abstract class JupyterActions extends Actions { starts, it creates this service. */ protected async api_call( - endpoint: string, + endpoint: JupyterApiEndpoint, query?: any, timeout_ms?: number, - ): Promise { + ) { if (this._state === "closed") { throw Error("closed -- jupyter actions -- api_call"); } diff --git a/src/packages/nats/service/project.ts b/src/packages/nats/service/project.ts index 55585dc666..cc92588f64 100644 --- a/src/packages/nats/service/project.ts +++ b/src/packages/nats/service/project.ts @@ -18,8 +18,20 @@ export function formatter({ compute_server_id = 0, project_id }) { ); } +export type JupyterApiEndpoint = + | "signal" + | "save_ipynb_file" + | "kernel_info" + | "more_output" + | "complete" + | "introspect" + | "store" + | "comm" + | "ipywidgets-get-buffer" + | "kernels"; + interface JupyterApiMessage { - endpoint: string; + endpoint: JupyterApiEndpoint; query?: any; } From 8691b751729c06dafaf1532ab1deb56c4ee0bd84 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 14 Feb 2025 20:25:09 +0000 Subject: [PATCH 201/281] nats: auth -- fix service permissions --- src/packages/server/nats/auth.ts | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index ad0499d0b1..c8b260c9d5 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -124,17 +124,15 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { goalPub.add("$JS.API.*.*.public"); goalPub.add("$JS.API.*.*.public.>"); goalPub.add("$JS.API.CONSUMER.MSG.NEXT.public.>"); + + // microservices info api + goalSub.add(`$SRV.>`); + goalPub.add(`$SRV.*`); if (userType == "account") { goalSub.add(`*.account-${userId}.>`); goalPub.add(`*.account-${userId}.>`); - // microservices api - goalSub.add(`$SRV.*.account-${userId}.>`); - goalSub.add(`$SRV.*.account-${userId}`); - goalSub.add(`$SRV.*`); - goalPub.add(`$SRV.*`); - // the account-specific kv stores goalPub.add(`$JS.API.*.*.KV_account-${userId}`); goalPub.add(`$JS.API.*.*.KV_account-${userId}.>`); @@ -151,12 +149,6 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { add(goalPub, pub); } } else if (userType == "project") { - // microservices api - goalSub.add(`$SRV.*.project-${userId}.>`); - goalSub.add(`$SRV.*.project-${userId}`); - goalSub.add(`$SRV.*`); - goalPub.add(`$SRV.*`); - const { pub, sub } = projectSubjects(userId); add(goalSub, sub); add(goalPub, pub); From 8159cae8c1aec3fae3675ea64626ef6eb5d77964 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 15 Feb 2025 12:20:15 +0000 Subject: [PATCH 202/281] nats sync: fix issue with changefeeds not working --- src/packages/server/nats/index.ts | 2 +- src/packages/sync/table/changefeed-nats.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts index cffb42b33e..3ed28f115c 100644 --- a/src/packages/server/nats/index.ts +++ b/src/packages/server/nats/index.ts @@ -1,7 +1,7 @@ import getLogger from "@cocalc/backend/logger"; import { initAPI } from "./api"; import { init as initDatabase } from "@cocalc/database/nats/changefeeds"; - +f const logger = getLogger("server:nats"); export default async function initNatsServer() { diff --git a/src/packages/sync/table/changefeed-nats.ts b/src/packages/sync/table/changefeed-nats.ts index 9a94abe799..01c2c02c6c 100644 --- a/src/packages/sync/table/changefeed-nats.ts +++ b/src/packages/sync/table/changefeed-nats.ts @@ -33,6 +33,7 @@ export class NatsChangefeed extends EventEmitter { atomic: true, immutable: false, }); + this.state = "connected"; this.interest(); this.startWatch(); const v = this.natsSynctable.get(); @@ -77,7 +78,7 @@ export class NatsChangefeed extends EventEmitter { return; } this.natsSynctable.on( - "changefeed", + "change", (_, { key, value: new_val, prev: old_val }) => { let x; if (new_val == null) { From a3bc1ea14b997edd8fc44c086b5c11de9a3afe05 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 15 Feb 2025 13:28:21 +0000 Subject: [PATCH 203/281] nats: sage worksheets - mainly supporting autoexpiring evaluations --- .../frontend/project/websocket/connect.ts | 4 +++ src/packages/nats/sync/general-kv.ts | 5 +++- src/packages/nats/sync/stream.ts | 4 +-- src/packages/nats/sync/synctable-kv.ts | 7 +++++ src/packages/nats/sync/synctable-stream.ts | 6 ++++ src/packages/nats/sync/synctable.ts | 4 ++- src/packages/project/sage_socket.ts | 6 ++-- src/packages/sync/editor/generic/evaluator.ts | 8 ++--- src/packages/sync/editor/generic/sync-doc.ts | 30 ++++++++++++++----- src/packages/util/misc.ts | 8 ++--- 10 files changed, 60 insertions(+), 22 deletions(-) diff --git a/src/packages/frontend/project/websocket/connect.ts b/src/packages/frontend/project/websocket/connect.ts index 2b04449565..8b60a1fd8d 100644 --- a/src/packages/frontend/project/websocket/connect.ts +++ b/src/packages/frontend/project/websocket/connect.ts @@ -252,6 +252,10 @@ async function connection_to_project0(project_id: string): Promise { conn.open(); }); + // conn.on("data", (data) => { + // console.log("project websocket received data", data); + // }); + return conn; } diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index 00715320aa..6b286e974c 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -522,7 +522,10 @@ export class GeneralKV extends EventEmitter { }); } } catch (err) { - if (err.code != "TIMEOUT") { + // code 10071 is for "JetStreamApiError: wrong last sequence", which is + // expected when there are multiple clients, since all of them try to impose + // limits up at once. + if (err.code != "TIMEOUT" && err.code != 10071) { console.log(`WARNING: expiring old messages - ${err}`); } } diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 89707cb368..076911265d 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -78,7 +78,7 @@ const EPHEMERAL_CONSUMER_THRESH = 5 * 60 * 1000; // the true limit is the minimum of the full NATS stream limits and // these limits. const ENFORCE_LIMITS_THROTTLE_MS = 3000; -interface FilteredStreamLimitOptions { +export interface FilteredStreamLimitOptions { // How many messages may be in a Stream, oldest messages will be removed // if the Stream exceeds this size. -1 for unlimited. max_msgs: number; @@ -556,7 +556,7 @@ export interface UserStreamOptions { name: string; account_id?: string; project_id?: string; - limits?: FilteredStreamLimitOptions; + limits?: Partial; start_seq?: number; noCache?: boolean; } diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 8306aee96f..3a10fabb86 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -15,6 +15,7 @@ import jsonStableStringify from "json-stable-stringify"; import { toKey } from "@cocalc/nats/util"; import { wait } from "@cocalc/util/async-wait"; import { fromJS, Map } from "immutable"; +import { type KVLimits } from "./general-kv"; export class SyncTableKV extends EventEmitter { public readonly table; @@ -27,6 +28,7 @@ export class SyncTableKV extends EventEmitter { private dkv?: DKV | DKO; private env; private getHook: Function; + private limits?: Partial; constructor({ query, @@ -35,6 +37,7 @@ export class SyncTableKV extends EventEmitter { project_id, atomic, immutable, + limits, }: { query; env: NatsEnv; @@ -42,11 +45,13 @@ export class SyncTableKV extends EventEmitter { project_id?: string; atomic?: boolean; immutable?: boolean; + limits?: Partial; }) { super(); this.atomic = !!atomic; this.getHook = immutable ? fromJS : (x) => x; this.query = query; + this.limits = limits; this.env = env; this.table = keys(query)[0]; if (query[this.table][0].string_id && query[this.table][0].project_id) { @@ -91,6 +96,7 @@ export class SyncTableKV extends EventEmitter { account_id: this.account_id, project_id: this.project_id, env: this.env, + limits: this.limits, }); } else { this.dkv = await createDko({ @@ -98,6 +104,7 @@ export class SyncTableKV extends EventEmitter { account_id: this.account_id, project_id: this.project_id, env: this.env, + limits: this.limits, }); } this.dkv.on("change", (x) => { diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts index 35cdca01ae..6f09e5acd6 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/nats/sync/synctable-stream.ts @@ -15,6 +15,7 @@ import { EventEmitter } from "events"; import { type NatsEnv } from "@cocalc/nats/types"; import { dstream, DStream } from "./dstream"; import { fromJS, Map } from "immutable"; +import { type FilteredStreamLimitOptions } from "./stream"; export type State = "disconnected" | "connected" | "closed"; @@ -39,6 +40,7 @@ export class SyncTableStream extends EventEmitter { private env; private dstream?: DStream; private getHook: Function; + private limits?: Partial; constructor({ query, @@ -46,16 +48,19 @@ export class SyncTableStream extends EventEmitter { account_id: _account_id, project_id, immutable, + limits, }: { query; env: NatsEnv; account_id?: string; project_id?: string; immutable?: boolean; + limits?: Partial; }) { super(); this.getHook = immutable ? fromJS : (x) => x; this.env = env; + this.limits = limits; const table = keys(query)[0]; this.table = table; if (table != "patches") { @@ -81,6 +86,7 @@ export class SyncTableStream extends EventEmitter { name, project_id: this.project_id, env: this.env, + limits: this.limits, }); this.dstream.on("change", (mesg) => { this.handle(mesg, true); diff --git a/src/packages/nats/sync/synctable.ts b/src/packages/nats/sync/synctable.ts index 279c2099d1..4512400f59 100644 --- a/src/packages/nats/sync/synctable.ts +++ b/src/packages/nats/sync/synctable.ts @@ -2,6 +2,8 @@ import { type NatsEnv } from "@cocalc/nats/types"; import { SyncTableKV } from "./synctable-kv"; import { SyncTableStream } from "./synctable-stream"; import { refCacheSync } from "@cocalc/util/refcache"; +import { type KVLimits } from "./general-kv"; +import { type FilteredStreamLimitOptions } from "./stream"; export type NatsSyncTable = SyncTableStream | SyncTableKV; @@ -32,6 +34,7 @@ interface Options { stream?: boolean; immutable?: boolean; // if true, then get/set works with immutable.js objects instead. noCache?: boolean; + limits?: Partial | Partial; } function createObject(options: Options) { @@ -45,5 +48,4 @@ function createObject(options: Options) { export const createSyncTable = refCacheSync({ createKey: (opts) => JSON.stringify({ ...opts, env: undefined }), createObject, - // name: "synctable-nats", }); diff --git a/src/packages/project/sage_socket.ts b/src/packages/project/sage_socket.ts index 6166547bb0..60e98cc82e 100644 --- a/src/packages/project/sage_socket.ts +++ b/src/packages/project/sage_socket.ts @@ -56,7 +56,7 @@ export async function get_sage_socket(): Promise { factor: 1.5, max_time: SAGE_SERVER_MAX_STARTUP_TIME_S * 1000, log(m) { - return winston.debug(`get_sage_socket: ${m}`); + winston.debug(`get_sage_socket: ${m}`); }, cb(err) { if (socket == null) { @@ -87,13 +87,13 @@ async function _get_sage_socket(): Promise { enable_mesg(sage_socket); sage_socket.write_mesg("json", message.start_session({ type: "sage" })); winston.debug( - "Waiting to read one JSON message back, which will describe the session...." + "Waiting to read one JSON message back, which will describe the session....", ); // TODO: couldn't this just hang forever :-( return new Promise((resolve) => { sage_socket.once("mesg", (_type, desc) => { winston.debug( - `Got message back from Sage server: ${common.json(desc)}` + `Got message back from Sage server: ${common.json(desc)}`, ); sage_socket.pid = desc.pid; resolve(sage_socket); diff --git a/src/packages/sync/editor/generic/evaluator.ts b/src/packages/sync/editor/generic/evaluator.ts index d98644287c..be39e9774b 100644 --- a/src/packages/sync/editor/generic/evaluator.ts +++ b/src/packages/sync/editor/generic/evaluator.ts @@ -354,10 +354,10 @@ export class Evaluator { dbg(`no outputs yet with key ${to_json(id)}`); const r = this.inputs_table.get(key); if (r == null) { - dbg("deleting from input?"); - throw Error("deleting from input not implemented"); - // happens when deleting from input table (if that is - // ever supported, e.g., for maybe trimming old evals...) + dbg("deleted old input"); + // This happens when deleting from input table (if that is + // ever supported, e.g., for maybe trimming old evals...). + // Nothing we need to do here. return; } const input = r.get("input"); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 44fe95675f..b2e34d0c30 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1275,7 +1275,12 @@ export class SyncDoc extends EventEmitter { ): Promise => { this.assert_not_closed("synctable"); const dbg = this.dbg("synctable"); - if (!this.ephemeral && this.persistent && this.data_server == "project") { + if ( + !this.useNats && + !this.ephemeral && + this.persistent && + this.data_server == "project" + ) { // persistent table in a non-ephemeral syncdoc, so ensure that table is // persisted to database (not just in memory). options = options.concat([{ persistent: true }]); @@ -1313,6 +1318,17 @@ export class SyncDoc extends EventEmitter { atomic: true, immutable: true, }); + } else if (this.useNats && (query.eval_inputs || query.eval_outputs)) { + synctable = await this.client.synctable_nats(query, { + obj: { + project_id: this.project_id, + path: this.path, + }, + stream: false, + atomic: true, + immutable: true, + limits: { max_age: 30000 }, + }); } else if (this.useNats) { synctable = await this.client.synctable_nats(query, { obj: { @@ -1886,7 +1902,7 @@ export class SyncDoc extends EventEmitter { this._save_patch(new_time, JSON.parse(obj.patch)) */ - private async init_evaluator(): Promise { + private init_evaluator = async () => { const dbg = this.dbg("init_evaluator"); const ext = filename_extension(this.path); if (ext !== "sagews") { @@ -1897,9 +1913,9 @@ export class SyncDoc extends EventEmitter { this.evaluator = new Evaluator(this, this.client, this.synctable); await this.evaluator.init(); dbg("done"); - } + }; - private async init_ipywidgets(): Promise { + private init_ipywidgets = async () => { const dbg = this.dbg("init_evaluator"); const ext = filename_extension(this.path); if (ext != "sage-jupyter2") { @@ -1914,9 +1930,9 @@ export class SyncDoc extends EventEmitter { ); await this.ipywidgets_state.init(); dbg("done"); - } + }; - private async init_cursors(): Promise { + private init_cursors = async () => { const dbg = this.dbg("init_cursors"); if (!this.cursors) { dbg("done -- do not care about cursors for this syncdoc."); @@ -2010,7 +2026,7 @@ export class SyncDoc extends EventEmitter { } dbg("done"); - } + }; private handle_cursors_change = (keys) => { if (this.state === "closed") { diff --git a/src/packages/util/misc.ts b/src/packages/util/misc.ts index b39a590374..63ab605a0d 100644 --- a/src/packages/util/misc.ts +++ b/src/packages/util/misc.ts @@ -1479,19 +1479,19 @@ export function retry_until_success(opts: { } if (err && opts.warn != null) { opts.warn( - `retry_until_success(${opts.name}) -- err=${JSON.stringify(err)}`, + `retry_until_success(${opts.name}) -- err=${err}`, ); } if (opts.log != null) { opts.log( - `retry_until_success(${opts.name}) -- err=${JSON.stringify(err)}`, + `retry_until_success(${opts.name}) -- err=${err}`, ); } if (opts.max_tries != null && opts.max_tries <= tries) { opts.cb?.( `maximum tries (=${ opts.max_tries - }) exceeded - last error ${JSON.stringify(err)}`, + }) exceeded - last error ${err}`, err, ); return; @@ -1507,7 +1507,7 @@ export function retry_until_success(opts: { opts.cb?.( `maximum time (=${ opts.max_time - }ms) exceeded - last error ${JSON.stringify(err)}`, + }ms) exceeded - last error ${err}`, err, ); return; From 9deb844ebc9fc9115281912e413d832a1a5c77f8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 15 Feb 2025 13:42:29 +0000 Subject: [PATCH 204/281] nats sync: expire ipywidgets --- src/packages/sync/editor/generic/sync-doc.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index b2e34d0c30..346682cc68 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1317,6 +1317,10 @@ export class SyncDoc extends EventEmitter { stream: false, atomic: true, immutable: true, + // for now just putting a 1-day limit on the ipywidgets table + // so we don't waste a ton of space. We need to also clear this + // table on halt, startup, etc. + limits: { max_age: 1000 * 60 * 60 * 24 }, }); } else if (this.useNats && (query.eval_inputs || query.eval_outputs)) { synctable = await this.client.synctable_nats(query, { From 92ad19d7bfd96c55bc8bed8ce96804bb81f276ab Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 15 Feb 2025 14:24:54 +0000 Subject: [PATCH 205/281] nats terminal: fixing some issues with initial load and sizing --- .../terminal-editor/connected-terminal.ts | 55 +++++++++++-------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 2e194c9a1a..9d83c0fecf 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -290,9 +290,13 @@ export class Terminal { conn.on("close", this.connect); conn.on("data", this._handle_data_from_project); conn.once("ready", () => { + delete this.last_geom; this.ignore_terminal_data = false; this.set_connection_status("connected"); this.scroll_to_bottom(); + this.terminal.refresh(0, this.terminal.rows - 1); + this.init_keyhandler(); + this.measure_size(); }); await conn.init(); } catch (err) { @@ -471,14 +475,18 @@ export class Terminal { } this.keyhandler_initialized = true; this.terminal.attachCustomKeyEventHandler((event) => { - /* - console.log("key", { - ctrlKey: event.ctrlKey, - metaKey: event.metaKey, - shiftKey: event.shiftKey, - key: event.key, - }); - */ + if (event.type !== "keydown") { + // ignore this + return true; + } + // console.log("key", { + // type: event.type, + // ctrlKey: event.ctrlKey, + // metaKey: event.metaKey, + // shiftKey: event.shiftKey, + // key: event.key, + // }); + // record that terminal is being actively used. this.last_active = Date.now(); this.ignore_terminal_data = false; @@ -487,11 +495,6 @@ export class Terminal { this.actions.unpause(this.id); } - if (event.type === "keypress") { - // ignore this - return true; - } - if ( (event.ctrlKey || event.metaKey) && event.shiftKey && @@ -640,6 +643,8 @@ export class Terminal { } }; + i; + // Stop ignoring terminal data... but ONLY once // the render buffer is also empty. async no_ignore(): Promise { @@ -852,19 +857,21 @@ export class Terminal { } measure_size(): void { - const geom = this.fitAddon.proposeDimensions(); - // console.log('measure_size', geom); - if (geom == null) return; - const { rows, cols } = geom; if (this.ignore_terminal_data) { - // during the initial render - this.terminal_resize({ rows, cols }); + // during initial load + return; } - if ( - this.last_geom !== undefined && - this.last_geom.rows === rows && - this.last_geom.cols === cols - ) { + const geom = this.fitAddon.proposeDimensions(); + // console.log("measure_size", { + // geom, + // ignore: this.ignore_terminal_data, + // last_geom: this.last_geom, + // }); + if (geom == null) { + return; + } + const { rows, cols } = geom; + if (this.last_geom?.rows === rows && this.last_geom?.cols === cols) { return; } this.last_geom = { rows, cols }; From d80b55dc942d6765d595e545bbcbd4a57def4694 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 15 Feb 2025 16:17:48 +0000 Subject: [PATCH 206/281] nats: rewriting more terminal functionality --- .../terminal-editor/connected-terminal.ts | 6 ++ .../nats-terminal-connection.ts | 76 ++++++++++----- src/packages/nats/project-api/index.ts | 3 - src/packages/nats/project-api/terminal.ts | 13 --- src/packages/nats/service/terminal.ts | 64 +++++++++++++ src/packages/nats/sync/open-files.ts | 9 ++ src/packages/project/nats/api/index.ts | 2 - src/packages/project/nats/api/terminal.ts | 8 -- src/packages/project/nats/open-files.ts | 67 ++++++++------ src/packages/project/nats/terminal.ts | 92 ++++++++++++++++--- 10 files changed, 246 insertions(+), 94 deletions(-) delete mode 100644 src/packages/nats/project-api/terminal.ts create mode 100644 src/packages/nats/service/terminal.ts delete mode 100644 src/packages/project/nats/api/terminal.ts diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 9d83c0fecf..e67d40e544 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -285,6 +285,12 @@ export class Terminal { openPaths: this.open_paths, closePaths: this.close_paths, compute_server_id: await this.getComputeServerId(), + options: { + command: this.command, + args: this.args, + cwd: this.workingDir, + env: this.actions.get_term_env(), + }, }); this.conn = conn as any; conn.on("close", this.connect); diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index c5ad3263ce..bb7dfebeec 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -6,51 +6,57 @@ import { uuid } from "@cocalc/util/misc"; import { delay } from "awaiting"; import { type DStream } from "@cocalc/nats/sync/dstream"; import { projectSubject } from "@cocalc/nats/names"; +import { + terminalService, + type TerminalService, +} from "@cocalc/nats/service/terminal"; +import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; const jc = JSONCodec(); const client = uuid(); +type State = "init" | "running" | "closed"; + export class NatsTerminalConnection extends EventEmitter { private project_id: string; //private compute_server_id: number; private path: string; private cmd_subject: string; - private state: null | "running" | "init" | "closed"; + private state: State = "init"; private stream?: DStream; - // keep = optional number of messages to retain between clients/sessions/view, i.e., - // "amount of history". This is global to all terminals in the project. - private keep?: number; private terminalResize; private openPaths; private closePaths; - private project; + private service: TerminalService; + private options?; constructor({ project_id, compute_server_id, path, - keep, terminalResize, openPaths, closePaths, + options, }: { project_id: string; compute_server_id: number; path: string; - keep?: number; terminalResize; openPaths; closePaths; + options?; }) { super(); this.project_id = project_id; //this.compute_server_id = compute_server_id; this.path = path; + this.options = options; + this.touchLoop({ project_id, path }); + this.service = terminalService({ project_id, path }); this.terminalResize = terminalResize; - this.keep = keep; this.openPaths = openPaths; this.closePaths = closePaths; - this.project = webapp_client.nats_client.projectApi({ project_id }); this.cmd_subject = projectSubject({ project_id, compute_server_id, @@ -82,36 +88,56 @@ export class NatsTerminalConnection extends EventEmitter { // invalid measurement -- ignore; https://github.com/sagemathinc/cocalc/issues/4158 and https://github.com/sagemathinc/cocalc/issues/4266 return; } + await this.service.call({ event: "size", rows, cols, client }); + } else if (data.cmd == "cwd") { + await this.service.call({ event: "cwd" }); + } else if (data.cmd == "boot") { + await this.service.call({ event: "boot", client }); + } else if (data.cmd == "kill") { + await this.service.call({ event: "kill" }); + } else { + throw Error(`todo -- implement cmd ${JSON.stringify(data)}`); } - await this.project.terminal.command({ path: this.path, ...data, client }); return; } - const f = async () => { - await this.project.terminal.write({ - path: this.path, - data, - keep: this.keep, - }); - }; - try { - await f(); - } catch (_err) { + await this.service.call({ event: "write", data }); + } catch (err) { + console.log(err); + // TODO: obviously wrong! A timeout would restart our poor terminal! await this.start(); - await f(); } }; - end = () => { + touchLoop = async ({ project_id, path }) => { + while (this.state != ("closed" as State)) { + try { + await webapp_client.touchOpenFile({ + project_id, + path, + }); + } catch (err) { + console.warn(err); + } + if (this.state == ("closed" as State)) { + break; + } + await delay(NATS_OPEN_FILE_TOUCH_INTERVAL); + } + }; + + close = () => { this.stream?.close(); delete this.stream; - // todo -- anything else? this.state = "closed"; }; + end = () => { + this.close(); + }; + private start = reuseInFlight(async () => { - // ensure running: - await this.project.terminal.create({ path: this.path }); + await this.service.call({ ...this.options, event: "create-session" }); }); private getStream = async () => { diff --git a/src/packages/nats/project-api/index.ts b/src/packages/nats/project-api/index.ts index 9e0f528b30..2bc684cc0f 100644 --- a/src/packages/nats/project-api/index.ts +++ b/src/packages/nats/project-api/index.ts @@ -1,19 +1,16 @@ import { type System, system } from "./system"; -import { type Terminal, terminal } from "./terminal"; import { type Editor, editor } from "./editor"; import { type Sync, sync } from "./sync"; import { handleErrorMessage } from "@cocalc/nats/util"; export interface ProjectApi { system: System; - terminal: Terminal; editor: Editor; sync: Sync; } const ProjectApiStructure = { system, - terminal, editor, sync, } as const; diff --git a/src/packages/nats/project-api/terminal.ts b/src/packages/nats/project-api/terminal.ts deleted file mode 100644 index fb6978e7c6..0000000000 --- a/src/packages/nats/project-api/terminal.ts +++ /dev/null @@ -1,13 +0,0 @@ -export const terminal = { - create: true, - restart: true, - command: true, - write: true, -}; - -export interface Terminal { - create: (params) => Promise<{ subject: string }>; - restart: ({ path }) => Promise; - command: ({ path, cmd, ...args }) => Promise; - write: ({ data, path }) => Promise; -} diff --git a/src/packages/nats/service/terminal.ts b/src/packages/nats/service/terminal.ts new file mode 100644 index 0000000000..9c13816a94 --- /dev/null +++ b/src/packages/nats/service/terminal.ts @@ -0,0 +1,64 @@ +/* +Service for controlling a terminal served from a project/compute server. +*/ + +import { natsService, NatsService } from "./typed"; + +type Response = any; + +export interface CreateSession { + event: "create-session"; + env?: { [key: string]: string }; + command?: string; + args?: string[]; + cwd?: string; +} + +export interface Write { + event: "write"; + data: string; +} + +export interface Restart { + event: "restart"; +} + +export interface CWD { + event: "cwd"; +} + +export interface Kill { + event: "kill"; +} + +export interface Size { + event: "size"; + rows: number; + cols: number; + client: string; +} + +export interface Boot { + event: "boot"; + client: string; +} + +export type Message = + | CreateSession + | Write + | Restart + | CWD + | Kill + | Size + | Boot; + +export type TerminalService = NatsService; + +export function terminalService({ project_id, path }) { + return natsService({ + project_id, + path, + service: "api", + description: "Terminal API", + }); +} diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 9f222445f1..32a68612ea 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -73,6 +73,12 @@ export class OpenFiles extends EventEmitter { constructor({ env, project_id, noAutosave, noCache }: Options) { super(); + if (!env) { + throw Error("env must be specified"); + } + if (!project_id) { + throw Error("project_id must be specified"); + } this.env = env; this.project_id = project_id; this.noAutosave = noAutosave; @@ -132,6 +138,9 @@ export class OpenFiles extends EventEmitter { // touch it to indicate that it is open. // updates timestamp and ensures open=true. touch = (path: string) => { + if (!path) { + throw Error("path must be specified"); + } const dkv = this.getDkv(); // n = sequence number to make sure a write happens, which updates // server assigned timestamp. diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index 10aa4946c1..0afaa563fe 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -107,13 +107,11 @@ async function handleApiRequest(request, mesg) { } import * as system from "./system"; -import * as terminal from "./terminal"; import * as editor from "./editor"; import * as sync from "./sync"; export const projectApi: ProjectApi = { system, - terminal, editor, sync, }; diff --git a/src/packages/project/nats/api/terminal.ts b/src/packages/project/nats/api/terminal.ts deleted file mode 100644 index 8fee037ef1..0000000000 --- a/src/packages/project/nats/api/terminal.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { - createTerminal as create, - restartTerminal as restart, - terminalCommand as command, - writeToTerminal as write, -} from "@cocalc/project/nats/terminal"; - -export { create, restart, command, write }; diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 9b8b341de0..5e13cc473f 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -16,13 +16,13 @@ Set env variables as in a project (see api/index.ts ), then in nodejs: DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:nats:open-files node > x = await require("@cocalc/project/nats/open-files").init(); Object.keys(x) -[ 'openFiles', 'openSyncDocs' ] +[ 'openFiles', 'openDocs', 'formatter', 'terminate' ] > x.openFiles.getAll(); -> Object.keys(x.openSyncDocs) +> Object.keys(x.openDocs) -> s = x.openSyncDocs['z4.tasks'] +> s = x.openDocs['z4.tasks'] // now you can directly work with the syncdoc for a given file, // but from the perspective of the project, not the browser! @@ -47,6 +47,8 @@ import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; import { filename_extension, original_path } from "@cocalc/util/misc"; import { get_blob_store } from "@cocalc/jupyter/blobs"; import { createFormatterService } from "./formatter"; +import { type TerminalService } from "@cocalc/nats/service/terminal"; +import { createTerminalService } from "./terminal"; // ensure nats connection stuff is initialized import "@cocalc/backend/nats"; @@ -55,6 +57,7 @@ const logger = getLogger("project:nats:open-files"); let openFiles: OpenFiles | null = null; let formatter: any = null; +const openDocs: { [path: string]: SyncDoc | TerminalService } = {}; export async function init() { logger.debug("init"); @@ -74,16 +77,16 @@ export async function init() { handleChange(entry); }); - formatter = await createFormatterService({ openSyncDocs }); + formatter = await createFormatterService({ openSyncDocs: openDocs }); // usefule for development - return { openFiles, openSyncDocs, formatter, terminate }; + return { openFiles, openDocs, formatter, terminate }; } export function terminate() { logger.debug("terminating open-files service"); - for (const path in openSyncDocs) { - closeSyncDoc(path); + for (const path in openDocs) { + closeDoc(path); } openFiles?.close(); openFiles = null; @@ -92,17 +95,13 @@ export function terminate() { formatter = null; } -const openSyncDocs: { [path: string]: SyncDoc } = {}; -// for dev -export { openSyncDocs }; - function getCutoff() { return new Date(Date.now() - 2.5 * NATS_OPEN_FILE_TOUCH_INTERVAL); } async function handleChange({ path, open, time }: OpenFileEntry) { logger.debug("handleChange", { path, open, time }); - const syncDoc = openSyncDocs[path]; + const syncDoc = openDocs[path]; const isOpenHere = syncDoc != null; // TODO: need another table with compute server mappings // const id = 0; // todo @@ -110,7 +109,7 @@ async function handleChange({ path, open, time }: OpenFileEntry) { // if (isOpenHere) { // // close it here // logger.debug("handleChange: closing", { path }); - // closeSyncDoc(path); + // closeDoc(path); // } // // no further responsibility // return; @@ -118,7 +117,7 @@ async function handleChange({ path, open, time }: OpenFileEntry) { if (!open) { if (isOpenHere) { logger.debug("handleChange: closing", { path }); - closeSyncDoc(path); + closeDoc(path); } return; } @@ -126,7 +125,7 @@ async function handleChange({ path, open, time }: OpenFileEntry) { if (!isOpenHere) { logger.debug("handleChange: opening", { path }); // users actively care about this file being opened HERE, but it isn't - openSyncDoc(path); + openDoc(path); } return; } @@ -147,7 +146,7 @@ async function closeIgnoredFilesLoop() { if (openFiles?.state != "connected") { return; } - const paths = Object.keys(openSyncDocs); + const paths = Object.keys(openDocs); if (paths.length == 0) { logger.debug("closeIgnoredFiles: no paths currently open"); continue; @@ -166,41 +165,49 @@ async function closeIgnoredFilesLoop() { supportAutoclose(entry.path) ) { logger.debug("closeIgnoredFiles: closing due to inactivity", entry); - closeSyncDoc(entry.path); + closeDoc(entry.path); } } } } -const closeSyncDoc = reuseInFlight(async (path: string) => { +const closeDoc = reuseInFlight(async (path: string) => { logger.debug("close", { path }); - const syncDoc = openSyncDocs[path]; - if (syncDoc == null) { + const doc = openDocs[path]; + if (doc == null) { return; } - delete openSyncDocs[path]; + delete openDocs[path]; try { - await syncDoc.close(); + await doc.close(); } catch (err) { - logger.debug(`WARNING -- issue closing syncdoc -- ${err}`); + logger.debug(`WARNING -- issue closing doc -- ${err}`); openFiles?.setError(path, err); } }); -const openSyncDoc = reuseInFlight(async (path: string) => { +const openDoc = reuseInFlight(async (path: string) => { // todo -- will be async and needs to handle SyncDB and all the config... - logger.debug("openSyncDoc", { path }); - const syncDoc = openSyncDocs[path]; - if (syncDoc != null) { + logger.debug("openDoc", { path }); + + const doc = openDocs[path]; + if (doc != null) { + return; + } + + if (path.endsWith(".term")) { + const service = createTerminalService(path); + openDocs[path] = service; return; } + const client = getClient(); const doctype = await getSyncDocType({ project_id, path, client, }); - logger.debug("openSyncDoc got", { path, doctype }); + logger.debug("openDoc got", { path, doctype }); let syncdoc; if (doctype.type == "string") { @@ -218,10 +225,10 @@ const openSyncDoc = reuseInFlight(async (path: string) => { client, }); } - openSyncDocs[path] = syncdoc; + openDocs[path] = syncdoc; syncdoc.on("error", (err) => { - closeSyncDoc(path); + closeDoc(path); openFiles?.setError(path, err); logger.debug(`syncdoc error -- ${err}`, path); }); diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index f35760beca..d1e4a0d4c1 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -16,6 +16,9 @@ import { readlink, realpath } from "node:fs/promises"; import { dstream, type DStream } from "@cocalc/project/nats/sync"; import { getSubject } from "./names"; import getConnection from "./connection"; +import { terminalService } from "@cocalc/nats/service/terminal"; +import { project_id } from "@cocalc/project/data"; +import { isEqual } from "lodash"; const logger = getLogger("server:nats:terminal"); @@ -29,6 +32,48 @@ const jc = JSONCodec(); const sessions: { [name: string]: Session } = {}; +export function createTerminalService(path: string) { + const service = terminalService({ path, project_id }); + service.listen(async (mesg) => { + if (mesg == null) { + throw Error("invalid message -- must not be null"); + } + if (mesg.event == "create-session") { + const note = await createTerminal({ ...mesg, path }); + return { status: "ok", note }; + } + const session = sessions[path]; + if (session == null) { + throw Error("no session"); + } + switch (mesg.event) { + case "write": + if (typeof mesg.data != "string") { + throw Error(`data must be a string -- ${JSON.stringify(mesg.data)}`); + } + return session.write(mesg.data); + case "restart": + return session.restart(); + case "size": + return session.setSize(mesg); + case "cwd": + return await session.getCwd(); + default: + // @ts-ignore + throw Error(`unknown message type ${mesg.event}`); + } + }); + return service; +} + +function closeTerminal(path: string) { + const cur = sessions[path]; + if (cur != null) { + cur.close(); + delete sessions[path]; + } +} + export const createTerminal = reuseInFlight( async (params) => { if (params == null) { @@ -38,12 +83,24 @@ export const createTerminal = reuseInFlight( if (!path) { throw Error("path must be specified"); } - if (sessions[path] == null) { - const nc = await getConnection(); - sessions[path] = new Session({ path, options, nc }); - await sessions[path].init(); + let note = ""; + const cur = sessions[path]; + if (cur != null) { + if (!isEqual(cur.options, options) || cur.state == "closed") { + // clean up -- we will make new one below + closeTerminal(path); + note += "Closed existing session. "; + } else { + // already have a working session with correct options + note += "Already have working session with same options. "; + return note; + } } - return { subject: sessions[path].subject }; + note += "Creating new session."; + const nc = await getConnection(); + sessions[path] = new Session({ path, options, nc }); + await sessions[path].init(); + return note; }, { createKey: (args) => { @@ -85,15 +142,13 @@ export async function terminalCommand({ path, cmd, ...args }) { } class Session { + public state: "running" | "off" | "closed" = "off"; + public options; private path: string; - private options; private pty?; private size?: { rows: number; cols: number }; - // the subject where we publish our output - public subject: string; private cmd_subject: string; - private state: "running" | "off" = "off"; - private stream: DStream; + private stream?: DStream; private streamName: string; private nc; @@ -120,6 +175,17 @@ class Session { await this.init(); }; + close = () => { + this.pty?.destroy(); + this.stream?.close(); + delete this.pty; + delete this.stream; + this.state = "closed"; + if (sessions[this.path] === this) { + delete sessions[this.path]; + } + }; + private getHome = () => { return process.env.HOME ?? "/home/user"; }; @@ -177,11 +243,11 @@ class Session { logger.debug("connect stream to pty"); this.pty.onData((data) => { this.handleBackendMessages(data); - this.stream.publish({ data }); + this.stream?.publish({ data }); }); this.pty.onExit((status) => { - this.stream.publish({ data: EXIT_MESSAGE }); - this.stream.publish({ ...status, exit: true }); + this.stream?.publish({ data: EXIT_MESSAGE }); + this.stream?.publish({ ...status, exit: true }); this.state = "off"; }); }; From 75801e38c02f98660c0e63de7a3efbe9333a006a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 15 Feb 2025 20:23:53 +0000 Subject: [PATCH 207/281] nats: different approach to typed service RPC with terminal - this feels good --- .../nats-terminal-connection.ts | 20 ++-- src/packages/nats/service/service.ts | 24 ++-- src/packages/nats/service/terminal.ts | 103 ++++++++++-------- src/packages/project/nats/open-files.ts | 6 +- src/packages/project/nats/terminal.ts | 96 ++++++++++------ 5 files changed, 145 insertions(+), 104 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index bb7dfebeec..99c374d13f 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -7,8 +7,8 @@ import { delay } from "awaiting"; import { type DStream } from "@cocalc/nats/sync/dstream"; import { projectSubject } from "@cocalc/nats/names"; import { - terminalService, - type TerminalService, + createTerminalClient, + type TerminalServiceApi, } from "@cocalc/nats/service/terminal"; import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; @@ -27,7 +27,7 @@ export class NatsTerminalConnection extends EventEmitter { private terminalResize; private openPaths; private closePaths; - private service: TerminalService; + private service: TerminalServiceApi; private options?; constructor({ @@ -53,7 +53,7 @@ export class NatsTerminalConnection extends EventEmitter { this.path = path; this.options = options; this.touchLoop({ project_id, path }); - this.service = terminalService({ project_id, path }); + this.service = createTerminalClient({ project_id, path }); this.terminalResize = terminalResize; this.openPaths = openPaths; this.closePaths = closePaths; @@ -88,20 +88,20 @@ export class NatsTerminalConnection extends EventEmitter { // invalid measurement -- ignore; https://github.com/sagemathinc/cocalc/issues/4158 and https://github.com/sagemathinc/cocalc/issues/4266 return; } - await this.service.call({ event: "size", rows, cols, client }); + await this.service.size({ rows, cols, client }); } else if (data.cmd == "cwd") { - await this.service.call({ event: "cwd" }); + await this.service.cwd(); } else if (data.cmd == "boot") { - await this.service.call({ event: "boot", client }); + await this.service.boot({ client }); } else if (data.cmd == "kill") { - await this.service.call({ event: "kill" }); + await this.service.kill(); } else { throw Error(`todo -- implement cmd ${JSON.stringify(data)}`); } return; } try { - await this.service.call({ event: "write", data }); + this.service.write(data); } catch (err) { console.log(err); // TODO: obviously wrong! A timeout would restart our poor terminal! @@ -137,7 +137,7 @@ export class NatsTerminalConnection extends EventEmitter { }; private start = reuseInFlight(async () => { - await this.service.call({ ...this.options, event: "create-session" }); + await this.service.create(this.options); }); private getStream = async () => { diff --git a/src/packages/nats/service/service.ts b/src/packages/nats/service/service.ts index e368bae093..5825158a83 100644 --- a/src/packages/nats/service/service.ts +++ b/src/packages/nats/service/service.ts @@ -13,6 +13,7 @@ an error too. import { Svcm } from "@nats-io/services"; import { type NatsEnv } from "@cocalc/nats/types"; import { sha1, trunc_middle } from "@cocalc/util/misc"; +import { getEnv } from "@cocalc/nats/client"; const DEFAULT_TIMEOUT = 5000; @@ -33,10 +34,8 @@ export interface ServiceCall extends ServiceDescription { export async function callNatsService(opts: ServiceCall): Promise { // console.log("callNatsService", opts); - if (opts.env == null) { - throw Error("NATS env must be specified"); - } - const { nc, jc } = opts.env; + const env = opts.env ?? (await getEnv()); + const { nc, jc } = env; const subject = serviceSubject(opts); let resp; const timeout = opts.timeout ?? DEFAULT_TIMEOUT; @@ -48,9 +47,8 @@ export async function callNatsService(opts: ServiceCall): Promise { if (err.name == "NatsError") { const p = opts.path ? `${trunc_middle(opts.path, 64)}:` : ""; if (err.code == "503") { - throw Error( - `Not Available: service ${p}${opts.service} is not available`, - ); + err.message = `Not Available: service ${p}${opts.service} is not available`; + throw err; } else if (err.code == "TIMEOUT") { throw Error( `Timeout: service ${p}${opts.service} did not respond for ${Math.round(timeout / 1000)} seconds`, @@ -142,10 +140,8 @@ export class NatsService { } init = async () => { - if (this.options.env == null) { - throw Error("NATS env must be specified"); - } - const svcm = new Svcm(this.options.env.nc); + const env = this.options.env ?? (await getEnv()); + const svcm = new Svcm(env.nc); const service = await svcm.add({ name: serviceName(this.options), @@ -158,10 +154,8 @@ export class NatsService { }; private listen = async () => { - if (this.options.env == null) { - throw Error("NATS env must be specified"); - } - const jc = this.options.env.jc; + const env = this.options.env ?? (await getEnv()); + const jc = env.jc; for await (const mesg of this.api) { const request = jc.decode(mesg.data) ?? ({} as any); // console.log("handle nats service call", request); diff --git a/src/packages/nats/service/terminal.ts b/src/packages/nats/service/terminal.ts index 9c13816a94..0d2ac18b35 100644 --- a/src/packages/nats/service/terminal.ts +++ b/src/packages/nats/service/terminal.ts @@ -2,63 +2,78 @@ Service for controlling a terminal served from a project/compute server. */ -import { natsService, NatsService } from "./typed"; +import { callNatsService, createNatsService } from "./service"; +import { delay } from "awaiting"; -type Response = any; +export interface TerminalServiceApi { + create: (opts: { + env?: { [key: string]: string }; + command?: string; + args?: string[]; + cwd?: string; + }) => Promise<{ success: "ok"; note?: string }>; -export interface CreateSession { - event: "create-session"; - env?: { [key: string]: string }; - command?: string; - args?: string[]; - cwd?: string; -} + write: (data: string) => Promise; -export interface Write { - event: "write"; - data: string; -} + restart: () => Promise; -export interface Restart { - event: "restart"; -} + cwd: () => Promise; -export interface CWD { - event: "cwd"; -} + kill: () => Promise; -export interface Kill { - event: "kill"; -} + size: (opts: { rows: number; cols: number; client: string }) => Promise; -export interface Size { - event: "size"; - rows: number; - cols: number; - client: string; + boot: (opts: { client: string }) => Promise; } -export interface Boot { - event: "boot"; - client: string; -} +const names = ["create", "write", "restart", "cwd", "kill", "size", "boot"]; -export type Message = - | CreateSession - | Write - | Restart - | CWD - | Kill - | Size - | Boot; +const service = "terminal"; -export type TerminalService = NatsService; +export function createTerminalClient({ project_id, path }) { + const C: Partial = {}; + for (const name of names) { + C[name] = async (...args) => { + const f = async () => + await callNatsService({ + project_id, + path, + service, + mesg: { name, args }, + }); + + let d = 100; + const start = Date.now(); + while (Date.now() - start < 15000) { + try { + return await f(); + } catch (err) { + if (err.code == "503") { + d = Math.min(3000, d * 1.3); + await delay(d); + continue; + } + throw err; + } + } + }; + } + return C as TerminalServiceApi; +} -export function terminalService({ project_id, path }) { - return natsService({ +export async function createTerminalServer({ + project_id, + path, + impl, +}: { + project_id: string; + path: string; + impl: TerminalServiceApi; +}) { + return await createNatsService({ project_id, path, - service: "api", - description: "Terminal API", + service, + handler: async (mesg) => await impl[mesg.name](...mesg.args), }); } diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 5e13cc473f..e0d8bd6ea7 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -47,7 +47,7 @@ import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; import { filename_extension, original_path } from "@cocalc/util/misc"; import { get_blob_store } from "@cocalc/jupyter/blobs"; import { createFormatterService } from "./formatter"; -import { type TerminalService } from "@cocalc/nats/service/terminal"; +import { type NatsService } from "@cocalc/nats/service/service"; import { createTerminalService } from "./terminal"; // ensure nats connection stuff is initialized @@ -57,7 +57,7 @@ const logger = getLogger("project:nats:open-files"); let openFiles: OpenFiles | null = null; let formatter: any = null; -const openDocs: { [path: string]: SyncDoc | TerminalService } = {}; +const openDocs: { [path: string]: SyncDoc | NatsService } = {}; export async function init() { logger.debug("init"); @@ -196,7 +196,7 @@ const openDoc = reuseInFlight(async (path: string) => { } if (path.endsWith(".term")) { - const service = createTerminalService(path); + const service = await createTerminalService(path); openDocs[path] = service; return; } diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index d1e4a0d4c1..cc5e01ab74 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -16,7 +16,7 @@ import { readlink, realpath } from "node:fs/promises"; import { dstream, type DStream } from "@cocalc/project/nats/sync"; import { getSubject } from "./names"; import getConnection from "./connection"; -import { terminalService } from "@cocalc/nats/service/terminal"; +import { createTerminalServer } from "@cocalc/nats/service/terminal"; import { project_id } from "@cocalc/project/data"; import { isEqual } from "lodash"; @@ -32,38 +32,70 @@ const jc = JSONCodec(); const sessions: { [name: string]: Session } = {}; -export function createTerminalService(path: string) { - const service = terminalService({ path, project_id }); - service.listen(async (mesg) => { - if (mesg == null) { - throw Error("invalid message -- must not be null"); - } - if (mesg.event == "create-session") { - const note = await createTerminal({ ...mesg, path }); - return { status: "ok", note }; - } - const session = sessions[path]; - if (session == null) { - throw Error("no session"); - } - switch (mesg.event) { - case "write": - if (typeof mesg.data != "string") { - throw Error(`data must be a string -- ${JSON.stringify(mesg.data)}`); - } - return session.write(mesg.data); - case "restart": - return session.restart(); - case "size": - return session.setSize(mesg); - case "cwd": - return await session.getCwd(); - default: - // @ts-ignore - throw Error(`unknown message type ${mesg.event}`); +export async function createTerminalService(path: string) { + let options: any = undefined; + const getSession = async (noCreate?: boolean) => { + const cur = sessions[path]; + if (cur == null) { + if (noCreate) { + throw Error("no terminal session"); + } + await createTerminal({ ...options, path }); + return sessions[path]; } - }); - return service; + return cur; + }; + const impl = { + create: async (opts: { + env?: { [key: string]: string }; + command?: string; + args?: string[]; + cwd?: string; + }): Promise<{ success: "ok"; note?: string }> => { + // save options to reuse. + options = opts; + const note = await createTerminal({ ...opts, path }); + return { success: "ok", note }; + }, + + write: async (data: string): Promise => { + if (typeof data != "string") { + throw Error(`data must be a string -- ${JSON.stringify(data)}`); + } + const session = await getSession(); + await session.write(data); + }, + + restart: async () => { + const session = await getSession(); + await session.restart(); + }, + + cwd: async () => { + const session = await getSession(); + return await session.getCwd(); + }, + + kill: async () => { + try { + const session = await getSession(true); + await session.close(); + } catch { + return; + } + }, + + size: async (opts: { rows: number; cols: number; client: string }) => { + const session = await getSession(); + session.setSize(opts); + }, + + boot: async (opts: { client: string }): Promise => { + console.log("boot", opts); + }, + }; + + return await createTerminalServer({ path, project_id, impl }); } function closeTerminal(path: string) { From 5df95758fb084d74bf5787867112f9c475f47182 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 15 Feb 2025 21:18:25 +0000 Subject: [PATCH 208/281] nats: yet another attempt at nice typed RPC api microservices - this has basically no redundancy, full typescript, etc.! --- .../frontend/project/websocket/api.ts | 6 +- src/packages/jupyter/kernel/websocket-api.ts | 4 +- src/packages/jupyter/redux/actions.ts | 9 +-- src/packages/jupyter/redux/project-actions.ts | 28 ++++++-- src/packages/nats/service/jupyter.ts | 57 ++++++++++++++++ src/packages/nats/service/project.ts | 53 ++++++++------- src/packages/nats/service/terminal.ts | 62 ++++++++---------- src/packages/nats/service/typed.ts | 65 ++++++++----------- src/packages/project/nats/formatter.ts | 21 +++--- 9 files changed, 181 insertions(+), 124 deletions(-) create mode 100644 src/packages/nats/service/jupyter.ts diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index e9b6f87a3a..a46e36c127 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -36,7 +36,7 @@ import type { ExecuteCodeOutput, ExecuteCodeOptions, } from "@cocalc/util/types/execute-code"; -import * as services from "@cocalc/nats/service/project"; +import { formatterClient } from "@cocalc/nats/service/project"; export class API { private conn; @@ -250,11 +250,11 @@ export class API { compute_server_id?: number, ): Promise => { const options: FormatterOptions = this.check_formatter_available(config); - const formatter = services.formatter({ + const client = formatterClient({ project_id: this.project_id, compute_server_id, }); - return await formatter.call({ path, options }); + return await client.formatter({ path, options }); }; formatter_string = async ( diff --git a/src/packages/jupyter/kernel/websocket-api.ts b/src/packages/jupyter/kernel/websocket-api.ts index 26d8461ad3..f099664d71 100644 --- a/src/packages/jupyter/kernel/websocket-api.ts +++ b/src/packages/jupyter/kernel/websocket-api.ts @@ -11,7 +11,7 @@ various messages related to working with Jupyter. import { get_existing_kernel } from "@cocalc/jupyter/kernel"; import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data"; import { bufferToBase64 } from "@cocalc/util/base64"; -import { type JupyterApiEndpoint } from "@cocalc/nats/service/project"; +import { type JupyterApiEndpoint } from "@cocalc/nats/service/jupyter"; export async function handleApiRequest( path: string, @@ -22,7 +22,7 @@ export async function handleApiRequest( if(endpoint == 'kernels') { return await get_kernel_data(); } - + // Now endpoints that do depend on a specific kernel. const kernel = get_existing_kernel(path); if (kernel == null) { diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 6c61fd52a5..f9b90fec8d 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -41,8 +41,8 @@ import { SyncDB } from "@cocalc/sync/editor/db/sync"; import type { Client } from "@cocalc/sync/client/types"; import latexEnvs from "@cocalc/util/latex-envs"; import { debounce } from "lodash"; -import { jupyter as natsJupyterService } from "@cocalc/nats/service/project"; -import { type JupyterApiEndpoint } from "@cocalc/nats/service/project"; +import { jupyterApiClient } from "@cocalc/nats/service/jupyter"; +import { type JupyterApiEndpoint } from "@cocalc/nats/service/jupyter"; const { close, required, defaults } = misc; @@ -204,11 +204,12 @@ export abstract class JupyterActions extends Actions { if (this._state === "closed") { throw Error("closed -- jupyter actions -- api_call"); } - const service = natsJupyterService({ + const client = jupyterApiClient({ project_id: this.project_id, path: this.path, + timeout: timeout_ms, }); - return await service.call({ endpoint, query }, timeout_ms); + return await client[endpoint](query); } protected dbg = (f: string) => { diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 4b04f81df3..6ebc641dc1 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -34,7 +34,7 @@ import { handleApiRequest } from "@cocalc/jupyter/kernel/websocket-api"; import { callback } from "awaiting"; import { get_blob_store } from "@cocalc/jupyter/blobs"; import { removeJupyterRedux } from "@cocalc/jupyter/kernel"; -import { jupyter as natsJupyterService } from "@cocalc/nats/service/project"; +import { createNatsJupyterService } from "@cocalc/nats/service/jupyter"; // see https://github.com/sagemathinc/cocalc/issues/8060 const MAX_OUTPUT_SAVE_DELAY = 30000; @@ -216,12 +216,30 @@ export class JupyterActions extends JupyterActions0 { throw Error("unable to initialize nats API"); } const path = this.path; - const service = natsJupyterService({ + // const f = async ({ endpoint, query }) => { + // return await handleApiRequest(path, endpoint, query); + // }); + const f = (endpoint) => { + return async (query) => { + return await handleApiRequest(path, endpoint, query); + }; + }; + const impl = { + signal: f("signal"), + save_ipynb_file: f("save_ipynb_file"), + kernel_info: f("kernel_info"), + more_output: f("more_output"), + complete: f("complete"), + introspect: f("introspect"), + store: f("store"), + comm: f("comm"), + "ipywidgets-get-buffer": f("ipywidgets-get-buffer"), + kernels: f("kernels"), + }; + const service = await createNatsJupyterService({ project_id: this.project_id, path, - }); - service.listen(async ({ endpoint, query }) => { - return await handleApiRequest(path, endpoint, query); + impl, }); this.syncdb.on("closed", () => { service.close(); diff --git a/src/packages/nats/service/jupyter.ts b/src/packages/nats/service/jupyter.ts new file mode 100644 index 0000000000..d075c5a84f --- /dev/null +++ b/src/packages/nats/service/jupyter.ts @@ -0,0 +1,57 @@ +/* +Services in a project. +*/ + +import { createServiceClient, createServiceHandler } from "./typed"; + +const service = "api"; + +interface JupyterApi { + signal: any; + save_ipynb_file: any; + kernel_info: any; + more_output: any; + complete: any; + introspect: any; + store: any; + comm: any; + "ipywidgets-get-buffer": any; + kernels: any; +} + +export type JupyterApiEndpoint = keyof JupyterApi; + +export function jupyterApiClient({ + project_id, + path, + timeout, +}: { + project_id: string; + path: string; + timeout?: number; +}) { + return createServiceClient({ + project_id, + path, + service, + timeout, + }); +} + +export async function createNatsJupyterService({ + path, + project_id, + impl, +}: { + project_id: string; + path: string; + impl: JupyterApi; +}) { + return await createServiceHandler({ + project_id, + path, + service, + impl, + description: "Jupyter notebook compute API", + }); +} diff --git a/src/packages/nats/service/project.ts b/src/packages/nats/service/project.ts index cc92588f64..fb0500a1fc 100644 --- a/src/packages/nats/service/project.ts +++ b/src/packages/nats/service/project.ts @@ -2,7 +2,7 @@ Services in a project. */ -import { natsService } from "./typed"; +import { createServiceClient, createServiceHandler } from "./typed"; import type { Options as FormatterOptions, @@ -12,36 +12,35 @@ import type { // TODO: we may change it to NOT take compute server and have this listening from // project and all compute servers... and have only the one with the file open // actually reply. -export function formatter({ compute_server_id = 0, project_id }) { - return natsService<{ path: string; options: FormatterOptions }, FormatResult>( - { project_id, compute_server_id, service: "formatter" }, - ); +interface FormatterApi { + formatter: (opts: { + path: string; + options: FormatterOptions; + }) => Promise; } -export type JupyterApiEndpoint = - | "signal" - | "save_ipynb_file" - | "kernel_info" - | "more_output" - | "complete" - | "introspect" - | "store" - | "comm" - | "ipywidgets-get-buffer" - | "kernels"; - -interface JupyterApiMessage { - endpoint: JupyterApiEndpoint; - query?: any; +export function formatterClient({ compute_server_id = 0, project_id }) { + return createServiceClient({ + project_id, + compute_server_id, + service: "formatter", + }); } -type JupyterApiResponse = any; - -export function jupyter({ project_id, path }) { - return natsService({ +export async function createFormatterService({ + compute_server_id = 0, + project_id, + impl, +}: { + project_id: string; + compute_server_id?: number; + impl: FormatterApi; +}) { + return await createServiceHandler({ project_id, - path, - service: "api", - description: "Jupyter notebook compute API", + compute_server_id, + service: "formatter", + description: "Code formatter API", + impl, }); } diff --git a/src/packages/nats/service/terminal.ts b/src/packages/nats/service/terminal.ts index 0d2ac18b35..bec36363f8 100644 --- a/src/packages/nats/service/terminal.ts +++ b/src/packages/nats/service/terminal.ts @@ -2,8 +2,7 @@ Service for controlling a terminal served from a project/compute server. */ -import { callNatsService, createNatsService } from "./service"; -import { delay } from "awaiting"; +import { createServiceClient, createServiceHandler } from "./typed"; export interface TerminalServiceApi { create: (opts: { @@ -26,39 +25,10 @@ export interface TerminalServiceApi { boot: (opts: { client: string }) => Promise; } -const names = ["create", "write", "restart", "cwd", "kill", "size", "boot"]; - const service = "terminal"; export function createTerminalClient({ project_id, path }) { - const C: Partial = {}; - for (const name of names) { - C[name] = async (...args) => { - const f = async () => - await callNatsService({ - project_id, - path, - service, - mesg: { name, args }, - }); - - let d = 100; - const start = Date.now(); - while (Date.now() - start < 15000) { - try { - return await f(); - } catch (err) { - if (err.code == "503") { - d = Math.min(3000, d * 1.3); - await delay(d); - continue; - } - throw err; - } - } - }; - } - return C as TerminalServiceApi; + return createServiceClient({ project_id, path, service }); } export async function createTerminalServer({ @@ -70,10 +40,34 @@ export async function createTerminalServer({ path: string; impl: TerminalServiceApi; }) { - return await createNatsService({ + return await createServiceHandler({ project_id, path, service, - handler: async (mesg) => await impl[mesg.name](...mesg.args), + description: "Terminal service.", + impl, }); } + +/* +import { delay } from "awaiting"; +async function callWithRetry(f, maxTime) { + let d = 100; + const start = Date.now(); + while (Date.now() - start < maxTime) { + try { + return await f(); + } catch (err) { + if (err.code == "503") { + d = Math.min(3000, d * 1.3); + if (Date.now() + d - start >= maxTime) { + throw err; + } + await delay(d); + continue; + } + throw err; + } + } +} +*/ diff --git a/src/packages/nats/service/typed.ts b/src/packages/nats/service/typed.ts index 0bcc387673..639ebd44a9 100644 --- a/src/packages/nats/service/typed.ts +++ b/src/packages/nats/service/typed.ts @@ -1,44 +1,31 @@ import { callNatsService, createNatsService } from "./service"; -import type { NatsService as NatsService0, Options } from "./service"; -import { getEnv } from "@cocalc/nats/client"; +import type { Options, ServiceCall } from "./service"; -export function natsService( - options: Omit, -) { - return new NatsService(options); +export function createServiceClient(options: Omit) { + return new Proxy( + {}, + { + get: (_, prop) => { + if (typeof prop !== "string") { + return undefined; + } + return async (...args) => { + return await callNatsService({ + ...options, + mesg: { name: prop, args }, + }); + }; + }, + }, + ) as Api; } -export class NatsService { - private service?: NatsService0; - private options: Omit; - - constructor(options: Omit) { - this.options = options; - } - - listen = async (handler: (mesg: Message) => Promise) => { - this.service = await createNatsService({ - ...this.options, - handler, - env: await getEnv(), - }); - return this.service; - }; - - close = () => { - this.service?.close(); - delete this.service; - // @ts-ignore - delete this.options; - }; - - call = async (mesg: Message, timeout?: number): Promise => { - const resp = await callNatsService({ - ...this.options, - env: await getEnv(), - timeout, - mesg, - }); - return resp as Response; - }; +export async function createServiceHandler({ + impl, + ...options +}: Omit & { impl: Api }) { + return await createNatsService({ + ...options, + handler: async (mesg) => await impl[mesg.name](...mesg.args), + }); } diff --git a/src/packages/project/nats/formatter.ts b/src/packages/project/nats/formatter.ts index 196d491f81..f839364ff6 100644 --- a/src/packages/project/nats/formatter.ts +++ b/src/packages/project/nats/formatter.ts @@ -3,7 +3,7 @@ File formatting service. */ import { run_formatter, type Options } from "../formatters"; -import * as services from "@cocalc/nats/service/project"; +import { createFormatterService as create } from "@cocalc/nats/service/project"; import { compute_server_id, project_id } from "@cocalc/project/data"; interface Message { @@ -11,14 +11,15 @@ interface Message { options: Options; } -const formatter = services.formatter({ compute_server_id, project_id }); - export async function createFormatterService({ openSyncDocs }) { - return formatter.listen(async (opts: Message) => { - const syncstring = openSyncDocs[opts.path]; - if (syncstring == null) { - throw Error(`"${opts.path}" is not opened`); - } - return await run_formatter({ ...opts, syncstring }); - }); + const impl = { + formatter: async (opts: Message) => { + const syncstring = openSyncDocs[opts.path]; + if (syncstring == null) { + throw Error(`"${opts.path}" is not opened`); + } + return await run_formatter({ ...opts, syncstring }); + }, + }; + return await create({ compute_server_id, project_id, impl }); } From 5ccf778eb6497f1a12f6569dcb23ab0d75c42c84 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 15 Feb 2025 23:36:41 +0000 Subject: [PATCH 209/281] nats jupyter api: more typing --- src/packages/jupyter/kernel/nats-service.ts | 130 ++++++++++++++++++ src/packages/jupyter/kernel/websocket-api.ts | 127 ----------------- src/packages/jupyter/redux/project-actions.ts | 82 +---------- .../jupyter/types/project-interface.ts | 34 +---- src/packages/jupyter/types/types.ts | 25 +--- src/packages/nats/service/jupyter.ts | 31 +++-- src/packages/util/jupyter/types.ts | 56 ++++++++ 7 files changed, 216 insertions(+), 269 deletions(-) create mode 100644 src/packages/jupyter/kernel/nats-service.ts delete mode 100644 src/packages/jupyter/kernel/websocket-api.ts diff --git a/src/packages/jupyter/kernel/nats-service.ts b/src/packages/jupyter/kernel/nats-service.ts new file mode 100644 index 0000000000..f13a58dbcc --- /dev/null +++ b/src/packages/jupyter/kernel/nats-service.ts @@ -0,0 +1,130 @@ +import { createNatsJupyterService } from "@cocalc/nats/service/jupyter"; +import { get_existing_kernel as getKernel } from "@cocalc/jupyter/kernel"; +import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data"; +import { bufferToBase64 } from "@cocalc/util/base64"; + +export async function initNatsService({ + path, + project_id, +}: { + path: string; + project_id: string; +}) { + const getExistingKernel = () => { + const kernel = getKernel(path); + if (kernel == null) { + throw Error(`no Jupyter kernel with path '${path}'`); + } + return kernel; + }; + + const impl = { + // Signal should be a string like "SIGINT", "SIGKILL". + signal: async (signal: string) => { + getKernel(path)?.signal(signal); + }, + + save_ipynb_file: async () => { + await getExistingKernel().save_ipynb_file(); + }, + + kernel_info: async () => { + return await getExistingKernel().kernel_info(); + }, + + more_output: async (id) => { + return getExistingKernel().more_output(id); + }, + + complete: async (opts) => { + return await getExistingKernel().complete(get_code_and_cursor_pos(opts)); + }, + + introspect: async (opts) => { + const { code, cursor_pos } = get_code_and_cursor_pos(opts); + let detail_level = 0; + if (opts.level != null) { + try { + detail_level = parseInt(opts.level); + if (detail_level < 0) { + detail_level = 0; + } else if (detail_level > 1) { + detail_level = 1; + } + } catch (err) {} + } + return await getExistingKernel().introspect({ + code, + cursor_pos, + detail_level, + }); + }, + store: async ({ + key, + value, + }: { + key: string; + value?: any; + }): Promise => { + const kernel = getExistingKernel(); + if (value === undefined) { + // undefined when getting the value + return kernel.store.get(key); + } else if (value === null) { + // null is used for deleting the value + kernel.store.delete(key); + return {}; + } else { + kernel.store.set(key, value); + return {}; + } + }, + comm: async (opts) => { + getExistingKernel().send_comm_message_to_kernel(opts); + }, + + "ipywidgets-get-buffer": async ({ model_id, buffer_path }) => { + const buffer = getExistingKernel().ipywidgetsGetBuffer( + model_id, + buffer_path, + ); + if (buffer == null) { + throw Error( + `no buffer for model=${model_id}, buffer_path=${JSON.stringify( + buffer_path, + )}`, + ); + } + return { buffer64: bufferToBase64(buffer) }; + }, + + kernels: get_kernel_data, + }; + return await createNatsJupyterService({ + project_id, + path, + impl, + }); +} + +function get_code_and_cursor_pos(opts): { + code: string; + cursor_pos: number; +} { + const code: string = opts.code; + if (!code) { + throw Error("must specify code"); + } + let cursor_pos: number; + if (opts.cursor_pos != null) { + try { + cursor_pos = parseInt(opts.cursor_pos); + } catch (error) { + cursor_pos = code.length; + } + } else { + cursor_pos = code.length; + } + + return { code, cursor_pos }; +} diff --git a/src/packages/jupyter/kernel/websocket-api.ts b/src/packages/jupyter/kernel/websocket-api.ts deleted file mode 100644 index f099664d71..0000000000 --- a/src/packages/jupyter/kernel/websocket-api.ts +++ /dev/null @@ -1,127 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Support for the project's websocket-based request/response API, which is used for handling -various messages related to working with Jupyter. -*/ - -import { get_existing_kernel } from "@cocalc/jupyter/kernel"; -import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data"; -import { bufferToBase64 } from "@cocalc/util/base64"; -import { type JupyterApiEndpoint } from "@cocalc/nats/service/jupyter"; - -export async function handleApiRequest( - path: string, - endpoint: JupyterApiEndpoint, - query?: any, -): Promise { - // First handle endpoints that do not depend on a specific kernel. - if(endpoint == 'kernels') { - return await get_kernel_data(); - } - - // Now endpoints that do depend on a specific kernel. - const kernel = get_existing_kernel(path); - if (kernel == null) { - if (endpoint == "signal") { - // It's not a serious problem to try to send a signal to a non-existent kernel. A no-op - // is completely reasonable, since you only send signals to kill or interrupt, and a non-existent - // kernel is already killed or interrupted. See https://github.com/sagemathinc/cocalc/issues/4420 - return {}; - } - throw Error(`api endpoint ${endpoint}: no kernel with path '${path}'`); - } - switch (endpoint) { - case "save_ipynb_file": - await kernel.save_ipynb_file(); - return {}; - - case "signal": - kernel.signal(query.signal); - return {}; - - case "kernel_info": - return await kernel.kernel_info(); - - case "more_output": - return kernel.more_output(query.id); - - case "complete": - return await kernel.complete(get_code_and_cursor_pos(query)); - - case "introspect": - const { code, cursor_pos } = get_code_and_cursor_pos(query); - let detail_level = 0; - if (query.level != null) { - try { - detail_level = parseInt(query.level); - if (detail_level < 0) { - detail_level = 0; - } else if (detail_level > 1) { - detail_level = 1; - } - } catch (err) {} - } - return await kernel.introspect({ - code, - cursor_pos, - detail_level, - }); - - case "store": - const { key, value } = query; - if (value === undefined) { - // undefined when getting the value - return kernel.store.get(key); - } else if (value === null) { - // null is used for deleting the value - kernel.store.delete(key); - return {}; - } else { - kernel.store.set(key, value); - return {}; - } - - case "comm": - return kernel.send_comm_message_to_kernel(query); - - case "ipywidgets-get-buffer": - const { model_id, buffer_path } = query; - const buffer = kernel.ipywidgetsGetBuffer(model_id, buffer_path); - if (buffer == null) { - throw Error( - `no buffer for model=${model_id}, buffer_path=${JSON.stringify( - buffer_path, - )}`, - ); - } - return { buffer64: bufferToBase64(buffer) }; - default: - throw Error(`unknown endpoint "${endpoint}"`); - } -} - -function get_code_and_cursor_pos(query: any): { - code: string; - cursor_pos: number; -} { - const code: string = query.code; - if (!code) { - throw Error("must specify code"); - } - let cursor_pos: number; - if (query.cursor_pos != null) { - try { - cursor_pos = parseInt(query.cursor_pos); - } catch (error) { - cursor_pos = code.length; - } - } else { - cursor_pos = code.length; - } - - return { code, cursor_pos }; -} diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 6ebc641dc1..8e6d99ff6d 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -30,11 +30,9 @@ import { decodeUUIDtoNum, isEncodedNumUUID, } from "@cocalc/util/compute/manager"; -import { handleApiRequest } from "@cocalc/jupyter/kernel/websocket-api"; -import { callback } from "awaiting"; import { get_blob_store } from "@cocalc/jupyter/blobs"; import { removeJupyterRedux } from "@cocalc/jupyter/kernel"; -import { createNatsJupyterService } from "@cocalc/nats/service/jupyter"; +import { initNatsService } from "@cocalc/jupyter/kernel/nats-service"; // see https://github.com/sagemathinc/cocalc/issues/8060 const MAX_OUTPUT_SAVE_DELAY = 30000; @@ -212,34 +210,9 @@ export class JupyterActions extends JupyterActions0 { } private initNatsApi = async () => { - if (this._client.createNatsService == null) { - throw Error("unable to initialize nats API"); - } - const path = this.path; - // const f = async ({ endpoint, query }) => { - // return await handleApiRequest(path, endpoint, query); - // }); - const f = (endpoint) => { - return async (query) => { - return await handleApiRequest(path, endpoint, query); - }; - }; - const impl = { - signal: f("signal"), - save_ipynb_file: f("save_ipynb_file"), - kernel_info: f("kernel_info"), - more_output: f("more_output"), - complete: f("complete"), - introspect: f("introspect"), - store: f("store"), - comm: f("comm"), - "ipywidgets-get-buffer": f("ipywidgets-get-buffer"), - kernels: f("kernels"), - }; - const service = await createNatsJupyterService({ + const service = await initNatsService({ project_id: this.project_id, - path, - impl, + path: this.path, }); this.syncdb.on("closed", () => { service.close(); @@ -1643,10 +1616,9 @@ export class JupyterActions extends JupyterActions0 { // either locally or via a remote compute server, depending on // whether this.remoteApiHandler is set (via the // register-to-handle-api event above). - const response = await this.handleApiRequest(data); spark.write({ event: "message", - data: { event: "api-response", response, id: data.id }, + data: { event: "error", error: "USE THE NEW NATS API!", id: data.id }, }); return; } @@ -1733,12 +1705,11 @@ export class JupyterActions extends JupyterActions0 { // output could be very BIG: // dbg(data); if (data.event == "api-request") { - const response = await this.handleApiRequest(data.request); try { await this.syncdb.sendMessageToProject({ event: "api-response", id: data.id, - response, + response: { error: "USE THE NEW NATS API!" }, }); } catch (err) { // this happens when the websocket is disconnected @@ -1748,49 +1719,6 @@ export class JupyterActions extends JupyterActions0 { } }; - private handleApiRequest = async (data) => { - if (this.remoteApiHandler != null) { - return await this.handleApiRequestViaRemoteApiHandler(data); - } - const dbg = this.dbg("handleApiRequest"); - const { path, endpoint, query } = data; - dbg("handling request in project", path); - try { - return await handleApiRequest(path, endpoint, query); - } catch (err) { - dbg("error -- ", err.message); - return { event: "error", message: err.message }; - } - }; - - private handleApiRequestViaRemoteApiHandler = async (data) => { - const dbg = this.dbg("handleApiRequestViaRemoteApiHandler"); - dbg(data?.path); - try { - if (!this.is_project) { - throw Error("BUG -- remote api requests only make sense in a project"); - } - if (this.remoteApiHandler == null) { - throw Error("BUG -- remote api handler not registered"); - } - // Send a message to the remote asking it to handle this api request, - // which calls the function handleMessageFromProject from above in that remote process. - const { id, spark, responseCallbacks } = this.remoteApiHandler; - spark.write({ - event: "message", - data: { event: "api-request", request: data, id }, - }); - const waitForResponse = (cb) => { - responseCallbacks[id] = cb; - }; - this.remoteApiHandler.id += 1; // increment sequential protocol message tracker id - return (await callback(waitForResponse)).response; - } catch (err) { - dbg("error -- ", err.message); - return { event: "error", message: err.message }; - } - }; - // Handle transient cell messages. handleTransientUpdate = (mesg) => { const display_id = mesg.content?.transient?.display_id; diff --git a/src/packages/jupyter/types/project-interface.ts b/src/packages/jupyter/types/project-interface.ts index 421fbcf124..8b37de388b 100644 --- a/src/packages/jupyter/types/project-interface.ts +++ b/src/packages/jupyter/types/project-interface.ts @@ -12,6 +12,8 @@ so that Typescript can meaningfully type check everything. */ import type { Channels } from "@nteract/messaging"; +import type { KernelInfo } from "@cocalc/util/jupyter/types"; +export type { KernelInfo }; // see https://gist.github.com/rsms/3744301784eb3af8ed80bc746bef5eeb#file-eventlistener-d-ts export interface EventEmitterInterface { @@ -79,38 +81,6 @@ export interface CodeExecutionEmitterInterface extends EventEmitterInterface { go(): Promise; } -interface CodeMirrorMode { - name: string; - version: number; -} - -interface HelpLink { - text: string; - url: string; -} - -interface LanguageInfo { - name: string; - version: string; - mimetype: string; - codemirror_mode: CodeMirrorMode; - pygments_lexer: string; - nbconvert_exporter: string; - file_extension: string; -} - -export interface KernelInfo { - nodejs_version: string; - start_time: number; - implementation_version: string; - banner: string; - protocol_version: string; - implementation: string; - status: string; - language_info: LanguageInfo; - help_links: HelpLink[]; -} - interface JupyterKernelInterfaceSpawnOpts { env: { [key: string]: string }; // environment variables } diff --git a/src/packages/jupyter/types/types.ts b/src/packages/jupyter/types/types.ts index 9e82db1614..5911b455db 100644 --- a/src/packages/jupyter/types/types.ts +++ b/src/packages/jupyter/types/types.ts @@ -5,6 +5,8 @@ import { LanguageModel } from "@cocalc/util/db-schema/llm-utils"; import type * as immutable from "immutable"; +import type { KernelSpec, KernelMetadata } from "@cocalc/util/jupyter/types"; +export type { KernelSpec, KernelMetadata }; export type NotebookMode = "edit" | "escape"; @@ -55,29 +57,6 @@ export type BackendState = | "starting" | "running"; -export interface KernelSpec { - name: string; - display_name: string; - language: string; - interrupt_mode: string; // usually "signal" - env: { [key: string]: string }; // usually {} - metadata?: KernelMetadata; - resource_dir: string; - argv: string[]; // comamnd+args, how the kernel will be launched -} - -export type KernelMetadata = { - // top level could contain a "cocalc" key, containing special settings understood by cocalc - cocalc?: { - priority?: number; // level 10 means it is important, on short list of choices, etc. 1 is low priority, for older versions - description: string; // Explains what the kernel is, eventually visible to the user - url: string; // a link to a website with more info about the kernel - } & { - // nested string/string key/value dictionary - [key: string]: string | Record; - }; -}; - export interface LLMTools { model: LanguageModel; setModel: (llm: LanguageModel) => void; diff --git a/src/packages/nats/service/jupyter.ts b/src/packages/nats/service/jupyter.ts index d075c5a84f..b8eacfa925 100644 --- a/src/packages/nats/service/jupyter.ts +++ b/src/packages/nats/service/jupyter.ts @@ -3,20 +3,31 @@ Services in a project. */ import { createServiceClient, createServiceHandler } from "./typed"; +import type { KernelInfo, KernelSpec } from "@cocalc/util/jupyter/types"; const service = "api"; interface JupyterApi { - signal: any; - save_ipynb_file: any; - kernel_info: any; - more_output: any; - complete: any; - introspect: any; - store: any; - comm: any; - "ipywidgets-get-buffer": any; - kernels: any; + signal: (signal: string) => Promise; + save_ipynb_file: () => Promise; + kernel_info: () => Promise; + more_output: (id: string) => Promise; + complete: (opts: { code: string; cursor_pos: number }) => Promise; + introspect: (opts: { code: string; cursor_pos: number }) => Promise; + store: (opts: { key: string; value?: any }) => Promise; + comm: (opts: { + msg_id: string; + comm_id: string; + target_name: string; + data: any; + buffers64?: string[]; + buffers?: Buffer[]; + }) => Promise; + "ipywidgets-get-buffer": (opts: { + model_id; + buffer_path; + }) => Promise<{ buffer64: string }>; + kernels: () => Promise; } export type JupyterApiEndpoint = keyof JupyterApi; diff --git a/src/packages/util/jupyter/types.ts b/src/packages/util/jupyter/types.ts index 99e7fd7955..e82a7222d4 100644 --- a/src/packages/util/jupyter/types.ts +++ b/src/packages/util/jupyter/types.ts @@ -3,3 +3,59 @@ export interface NbconvertParams { directory?: string; timeout?: number; // in seconds! } + +interface HelpLink { + text: string; + url: string; +} + +interface CodeMirrorMode { + name: string; + version: number; +} + +interface LanguageInfo { + name: string; + version: string; + mimetype: string; + codemirror_mode: CodeMirrorMode; + pygments_lexer: string; + nbconvert_exporter: string; + file_extension: string; +} + +export interface KernelInfo { + nodejs_version: string; + start_time: number; + implementation_version: string; + banner: string; + protocol_version: string; + implementation: string; + status: string; + language_info: LanguageInfo; + help_links: HelpLink[]; +} + + +export interface KernelSpec { + name: string; + display_name: string; + language: string; + interrupt_mode: string; // usually "signal" + env: { [key: string]: string }; // usually {} + metadata?: KernelMetadata; + resource_dir: string; + argv: string[]; // comamnd+args, how the kernel will be launched +} + +export type KernelMetadata = { + // top level could contain a "cocalc" key, containing special settings understood by cocalc + cocalc?: { + priority?: number; // level 10 means it is important, on short list of choices, etc. 1 is low priority, for older versions + description: string; // Explains what the kernel is, eventually visible to the user + url: string; // a link to a website with more info about the kernel + } & { + // nested string/string key/value dictionary + [key: string]: string | Record; + }; +}; \ No newline at end of file From 0e932d16a17bdc3eb7059cf895cab24feec7efbe Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 16 Feb 2025 01:12:54 +0000 Subject: [PATCH 210/281] nats terminal: properly implementing browser side --- src/packages/frontend/client/client.ts | 4 +- .../terminal-editor/connected-terminal.ts | 6 +- .../nats-terminal-connection.ts | 110 +++++++++--------- src/packages/nats/service/service.ts | 59 +++++++++- src/packages/nats/service/terminal.ts | 84 ++++++++----- src/packages/project/nats/terminal.ts | 82 +++++++------ 6 files changed, 219 insertions(+), 126 deletions(-) diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 7aa29791dd..daf9807d8b 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -39,6 +39,7 @@ import type { } from "@cocalc/nats/service"; import type { NatsEnvFunction } from "@cocalc/nats/types"; import { setNatsClient } from "@cocalc/nats/client"; +import { randomId } from "@cocalc/nats/names"; // This DEBUG variable comes from webpack: declare const DEBUG; @@ -63,6 +64,7 @@ export type AsyncCall = (opts: object) => Promise; export interface WebappClient extends EventEmitter { account_id?: string; + browser_id: string; stripe: StripeClient; project_collaborators: ProjectCollaborators; messages: Messages; @@ -147,6 +149,7 @@ Connection events: class Client extends EventEmitter implements WebappClient { account_id: string = Cookies.get(ACCOUNT_ID_COOKIE); + browser_id: string = randomId(); stripe: StripeClient; project_collaborators: ProjectCollaborators; messages: Messages; @@ -218,7 +221,6 @@ class Client extends EventEmitter implements WebappClient { return (..._) => {}; }; } - this.hub_client = bind_methods(new HubClient(this)); this.is_signed_in = this.hub_client.is_signed_in.bind(this.hub_client); this.is_connected = this.hub_client.is_connected.bind(this.hub_client); diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index e67d40e544..453e4506e9 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -284,7 +284,7 @@ export class Terminal { terminalResize: this.terminal_resize, openPaths: this.open_paths, closePaths: this.close_paths, - compute_server_id: await this.getComputeServerId(), + measureSize: this.measure_size, options: { command: this.command, args: this.args, @@ -862,7 +862,7 @@ export class Terminal { return this.terminal.options[option]; } - measure_size(): void { + measure_size = (): void => { if (this.ignore_terminal_data) { // during initial load return; @@ -882,7 +882,7 @@ export class Terminal { } this.last_geom = { rows, cols }; this.conn_write({ cmd: "size", rows, cols }); - } + }; copy(): void { const sel: string = this.terminal.getSelection(); diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 99c374d13f..390b71ec3d 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -1,68 +1,58 @@ import { webapp_client } from "@cocalc/frontend/webapp-client"; import { EventEmitter } from "events"; -import { JSONCodec } from "nats.ws"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { uuid } from "@cocalc/util/misc"; import { delay } from "awaiting"; import { type DStream } from "@cocalc/nats/sync/dstream"; -import { projectSubject } from "@cocalc/nats/names"; import { createTerminalClient, type TerminalServiceApi, + createBrowserService, + SIZE_TIMEOUT_MS, } from "@cocalc/nats/service/terminal"; import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; -const jc = JSONCodec(); -const client = uuid(); - type State = "init" | "running" | "closed"; export class NatsTerminalConnection extends EventEmitter { private project_id: string; - //private compute_server_id: number; private path: string; - private cmd_subject: string; private state: State = "init"; private stream?: DStream; private terminalResize; private openPaths; private closePaths; - private service: TerminalServiceApi; + private api: TerminalServiceApi; + private service?; private options?; constructor({ project_id, - compute_server_id, path, terminalResize, openPaths, closePaths, options, + measureSize, }: { project_id: string; - compute_server_id: number; path: string; terminalResize; openPaths; closePaths; options?; + measureSize?; }) { super(); this.project_id = project_id; - //this.compute_server_id = compute_server_id; this.path = path; this.options = options; this.touchLoop({ project_id, path }); - this.service = createTerminalClient({ project_id, path }); + this.sizeLoop(measureSize); + this.api = createTerminalClient({ project_id, path }); + this.createBrowserService(); this.terminalResize = terminalResize; this.openPaths = openPaths; this.closePaths = closePaths; - this.cmd_subject = projectSubject({ - project_id, - compute_server_id, - service: "terminal-cmd", - path, - }); } write = async (data) => { @@ -88,20 +78,24 @@ export class NatsTerminalConnection extends EventEmitter { // invalid measurement -- ignore; https://github.com/sagemathinc/cocalc/issues/4158 and https://github.com/sagemathinc/cocalc/issues/4266 return; } - await this.service.size({ rows, cols, client }); + await this.api.size({ + rows, + cols, + browser_id: webapp_client.browser_id, + }); } else if (data.cmd == "cwd") { - await this.service.cwd(); + await this.api.cwd(); } else if (data.cmd == "boot") { - await this.service.boot({ client }); + await this.api.boot({ browser_id: webapp_client.browser_id }); } else if (data.cmd == "kill") { - await this.service.kill(); + await this.api.kill(); } else { throw Error(`todo -- implement cmd ${JSON.stringify(data)}`); } return; } try { - this.service.write(data); + this.api.write(data); } catch (err) { console.log(err); // TODO: obviously wrong! A timeout would restart our poor terminal! @@ -126,9 +120,19 @@ export class NatsTerminalConnection extends EventEmitter { } }; + sizeLoop = async (measureSize) => { + while (this.state != ("closed" as State)) { + measureSize(); + await delay(SIZE_TIMEOUT_MS / 1.3); + } + }; + close = () => { this.stream?.close(); delete this.stream; + this.service?.close(); + delete this.service; + this.api.close(webapp_client.browser_id); this.state = "closed"; }; @@ -137,7 +141,7 @@ export class NatsTerminalConnection extends EventEmitter { }; private start = reuseInFlight(async () => { - await this.service.create(this.options); + await this.api.create(this.options); }); private getStream = async () => { @@ -154,37 +158,6 @@ export class NatsTerminalConnection extends EventEmitter { await this.start(); this.stream = await this.getStream(); this.consumeDataStream(); - this.subscribeToCommands(); - }; - - private subscribeToCommands = async () => { - const nc = await webapp_client.nats_client.getConnection(); - const sub = nc.subscribe(this.cmd_subject); - for await (const mesg of sub) { - if (this.state == "closed") { - return; - } - this.handleCommand(mesg); - } - }; - - private handleCommand = async (mesg) => { - const x = jc.decode(mesg.data) as any; - switch (x.cmd) { - case "size": - this.terminalResize(x); - return; - case "message": - if (x.payload?.event == "open") { - this.openPaths(x.payload.paths); - } else if (x.payload?.event == "close") { - this.closePaths(x.payload.paths); - } - return; - default: - console.log("TODO -- unhandled message from project:", x); - return; - } }; private handleStreamMessage = (mesg) => { @@ -214,4 +187,29 @@ export class NatsTerminalConnection extends EventEmitter { this.emit("ready"); } }; + + private createBrowserService = async () => { + const impl = { + command: async ({ event, paths }): Promise => { + if (event == "open") { + this.openPaths(paths); + } else if (event == "close") { + this.closePaths(paths); + } + }, + + kick: async ({ browser_id }) => { + console.log(`everyone but ${browser_id} must go!`); + }, + + size: async ({ rows, cols }) => { + this.terminalResize({ rows, cols }); + }, + }; + this.service = await createBrowserService({ + project_id: this.project_id, + path: this.path, + impl, + }); + }; } diff --git a/src/packages/nats/service/service.ts b/src/packages/nats/service/service.ts index 5825158a83..d1decc35b3 100644 --- a/src/packages/nats/service/service.ts +++ b/src/packages/nats/service/service.ts @@ -14,16 +14,24 @@ import { Svcm } from "@nats-io/services"; import { type NatsEnv } from "@cocalc/nats/types"; import { sha1, trunc_middle } from "@cocalc/util/misc"; import { getEnv } from "@cocalc/nats/client"; +import { randomId } from "@cocalc/nats/names"; const DEFAULT_TIMEOUT = 5000; export interface ServiceDescription { service: string; + project_id?: string; - account_id?: string; compute_server_id?: number; + + account_id?: string; + browser_id?: string; + path?: string; description?: string; + + // if true and multiple servers are setup in same "location", then they ALL get to respond (sender gets first response). + all?: boolean; } export interface ServiceCall extends ServiceDescription { @@ -76,41 +84,56 @@ export type CreateNatsServiceFunction = typeof createNatsService; export function serviceSubject({ service, + account_id, + browser_id, + project_id, compute_server_id, + path, }: ServiceDescription): string { let segments; + path = path ? sha1(path) : "-"; if (!project_id && !account_id) { segments = ["public", service]; + } else if (account_id) { + segments = [ + "services", + `account-${account_id}`, + browser_id ?? "-", + project_id ?? "-", + path ?? "-", + service, + ]; } else if (project_id) { segments = [ "services", `project-${project_id}`, compute_server_id ?? "-", service, - path ? sha1(path) : "-", + path, ]; - } else if (account_id) { - segments = ["services", `account-${account_id}`, service]; } return segments.join("."); } export function serviceName({ service, + account_id, + browser_id, + project_id, compute_server_id, }: ServiceDescription): string { let segments; if (!project_id && !account_id) { segments = [service]; + } else if (account_id) { + segments = [`account-${account_id}`, browser_id ?? "-", service]; } else if (project_id) { segments = [`project-${project_id}`, compute_server_id ?? "-", service]; - } else if (account_id) { - segments = [`account-${account_id}`, service]; } return segments.join("-"); } @@ -147,6 +170,7 @@ export class NatsService { name: serviceName(this.options), version: this.options.version ?? "0.0.1", description: serviceDescription(this.options), + queue: this.options.all ? randomId() : "0", }); this.api = service.addEndpoint("api", { subject: this.subject }); @@ -177,3 +201,26 @@ export class NatsService { delete this.options; }; } + +/* +import { delay } from "awaiting"; +async function callWithRetry(f, maxTime) { + let d = 100; + const start = Date.now(); + while (Date.now() - start < maxTime) { + try { + return await f(); + } catch (err) { + if (err.code == "503") { + d = Math.min(3000, d * 1.3); + if (Date.now() + d - start >= maxTime) { + throw err; + } + await delay(d); + continue; + } + throw err; + } + } +} +*/ diff --git a/src/packages/nats/service/terminal.ts b/src/packages/nats/service/terminal.ts index bec36363f8..025031239e 100644 --- a/src/packages/nats/service/terminal.ts +++ b/src/packages/nats/service/terminal.ts @@ -4,6 +4,10 @@ Service for controlling a terminal served from a project/compute server. import { createServiceClient, createServiceHandler } from "./typed"; +export const SIZE_TIMEOUT_MS = 45000; + +// API that runs under Node.js in linux: + export interface TerminalServiceApi { create: (opts: { env?: { [key: string]: string }; @@ -20,15 +24,24 @@ export interface TerminalServiceApi { kill: () => Promise; - size: (opts: { rows: number; cols: number; client: string }) => Promise; + size: (opts: { + rows: number; + cols: number; + browser_id: string; + }) => Promise; - boot: (opts: { client: string }) => Promise; -} + boot: (opts: { browser_id: string }) => Promise; -const service = "terminal"; + // send when this client is leaving. + close: (browser_id: string) => Promise; +} export function createTerminalClient({ project_id, path }) { - return createServiceClient({ project_id, path, service }); + return createServiceClient({ + project_id, + path, + service: "project-api", + }); } export async function createTerminalServer({ @@ -43,31 +56,48 @@ export async function createTerminalServer({ return await createServiceHandler({ project_id, path, - service, + service: "project-api", description: "Terminal service.", impl, }); } -/* -import { delay } from "awaiting"; -async function callWithRetry(f, maxTime) { - let d = 100; - const start = Date.now(); - while (Date.now() - start < maxTime) { - try { - return await f(); - } catch (err) { - if (err.code == "503") { - d = Math.min(3000, d * 1.3); - if (Date.now() + d - start >= maxTime) { - throw err; - } - await delay(d); - continue; - } - throw err; - } - } +// API that runs in the browser: + +export interface TerminalBrowserApi { + // command is used for things like "open foo.txt" in the terminal. + command: (mesg) => Promise; + + // used for kicking user out of the terminal + kick: (opts: { browser_id: string }) => Promise; + + // tell browser to change its size + size: (opts: { rows: number; cols: number }) => Promise; +} + +export function createBrowserClient({ project_id, path }) { + return createServiceClient({ + project_id, + path, + service: "browser-api", + }); +} + +export async function createBrowserService({ + project_id, + path, + impl, +}: { + project_id: string; + path: string; + impl: TerminalBrowserApi; +}) { + return await createServiceHandler({ + project_id, + path, + service: "browser-api", + description: "Browser Terminal service.", + all: true, + impl, + }); } -*/ diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index cc5e01ab74..f8f126e76c 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -10,13 +10,14 @@ import { path_split } from "@cocalc/util/misc"; import { console_init_filename, len } from "@cocalc/util/misc"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { JSONCodec } from "nats"; import { getLogger } from "@cocalc/project/logger"; import { readlink, realpath } from "node:fs/promises"; import { dstream, type DStream } from "@cocalc/project/nats/sync"; -import { getSubject } from "./names"; -import getConnection from "./connection"; -import { createTerminalServer } from "@cocalc/nats/service/terminal"; +import { + createTerminalServer, + createBrowserClient, + SIZE_TIMEOUT_MS, +} from "@cocalc/nats/service/terminal"; import { project_id } from "@cocalc/project/data"; import { isEqual } from "lodash"; @@ -28,9 +29,7 @@ const INFINITY = 999999; const HISTORY_LIMIT_BYTES = 20000; -const jc = JSONCodec(); - -const sessions: { [name: string]: Session } = {}; +const sessions: { [path: string]: Session } = {}; export async function createTerminalService(path: string) { let options: any = undefined; @@ -85,14 +84,18 @@ export async function createTerminalService(path: string) { } }, - size: async (opts: { rows: number; cols: number; client: string }) => { + size: async (opts: { rows: number; cols: number; browser_id: string }) => { const session = await getSession(); session.setSize(opts); }, - boot: async (opts: { client: string }): Promise => { + boot: async (opts: { browser_id: string }): Promise => { console.log("boot", opts); }, + + close: async (browser_id: string) => { + sessions[path]?.browserLeaving(browser_id); + }, }; return await createTerminalServer({ path, project_id, impl }); @@ -129,8 +132,7 @@ export const createTerminal = reuseInFlight( } } note += "Creating new session."; - const nc = await getConnection(); - sessions[path] = new Session({ path, options, nc }); + sessions[path] = new Session({ path, options }); await sessions[path].init(); return note; }, @@ -179,18 +181,19 @@ class Session { private path: string; private pty?; private size?: { rows: number; cols: number }; - private cmd_subject: string; + private browserApi: ReturnType; private stream?: DStream; private streamName: string; - private nc; + private clientSizes: { + [browser_id: string]: { rows: number; cols: number; time: number }; + } = {}; - constructor({ path, options, nc }) { + constructor({ path, options }) { logger.debug("create session ", { path, options }); this.path = path; + this.browserApi = createBrowserClient({ project_id, path }); this.options = options; - this.cmd_subject = getSubject({ service: "terminal-cmd", path }); this.streamName = `terminal-${path}`; - this.nc = nc; } write = async (data) => { @@ -216,6 +219,7 @@ class Session { if (sessions[this.path] === this) { delete sessions[this.path]; } + this.clientSizes = {}; }; private getHome = () => { @@ -284,24 +288,21 @@ class Session { }); }; - private publishCommand = (mesg) => { - this.nc.publish(this.cmd_subject, jc.encode(mesg)); - }; - - private clientSizes = {}; setSize = ({ - client, + browser_id, rows, cols, }: { - client: string; + browser_id: string; rows: number; cols: number; }) => { - //this.clientSizes[client] = { rows, cols }; - // just doing this silly hack for now -- we need to redo this algorithm to instead - // query all clients and when relevant, since no notion of connection. - this.clientSizes = { [client]: { rows, cols } }; + this.clientSizes[browser_id] = { rows, cols, time: Date.now() }; + this.resize(); + }; + + browserLeaving = (browser_id: string) => { + delete this.clientSizes[browser_id]; this.resize(); }; @@ -318,8 +319,8 @@ class Session { logger.debug("resize", "new size", rows, cols); try { this.setSizePty({ rows, cols }); - // broadcast out new size - this.publishCommand({ cmd: "size", rows, cols }); + // tell browsers about out new size + this.browserApi.size({ rows, cols }); } catch (err) { logger.debug("terminal channel -- WARNING: unable to resize term", err); } @@ -344,7 +345,12 @@ class Session { } let rows: number = INFINITY; let cols: number = INFINITY; + const cutoff = Date.now() - SIZE_TIMEOUT_MS; for (const id in sizes) { + if ((sizes[id].time ?? 0) <= cutoff) { + delete sizes[id]; + continue; + } if (sizes[id].rows) { // if, since 0 rows or 0 columns means *ignore*. rows = Math.min(rows, sizes[id].rows); @@ -377,8 +383,6 @@ class Session { /* parse out messages like this: \x1b]49;"valid JSON string here"\x07 and format and send them via our json channel. - NOTE: such messages also get sent via the - normal channel, but ignored by the client. */ if (this.backendMessagesState === "none") { const i = data.indexOf("\x1b]49;"); @@ -405,16 +409,28 @@ class Session { logger.debug( `handle_backend_message: parsing JSON payload ${JSON.stringify(s)}`, ); + let mesg; try { - const payload = JSON.parse(s); - this.publishCommand({ cmd: "message", payload }); + mesg = JSON.parse(s); } catch (err) { logger.warn( `handle_backend_message: error sending JSON payload ${JSON.stringify( s, )}, ${err}`, ); + return; } + (async () => { + try { + await this.browserApi.command(mesg); + } catch (err) { + // could fail, e.g., if there are no browser clients suddenly. + logger.debug( + "WARNING: problem sending command to browser clients", + err, + ); + } + })(); } }; } From f08bc330bdda07ba4f455e36f915c8f6900277eb Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 16 Feb 2025 01:23:02 +0000 Subject: [PATCH 211/281] terminal: mostly implement "kick" --- .../terminal-editor/connected-terminal.ts | 2 +- .../terminal-editor/nats-terminal-connection.ts | 17 +++++++++++++---- src/packages/nats/service/terminal.ts | 8 +++----- src/packages/project/nats/terminal.ts | 4 ---- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 453e4506e9..0b57d43e8b 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -800,7 +800,7 @@ export class Terminal { } kick_other_users_out(): void { - this.conn_write({ cmd: "boot" }); + this.conn.kick(); } kill(): void { diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 390b71ec3d..fbd7f40307 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -8,6 +8,7 @@ import { type TerminalServiceApi, createBrowserService, SIZE_TIMEOUT_MS, + createBrowserClient, } from "@cocalc/nats/service/terminal"; import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; @@ -85,8 +86,6 @@ export class NatsTerminalConnection extends EventEmitter { }); } else if (data.cmd == "cwd") { await this.api.cwd(); - } else if (data.cmd == "boot") { - await this.api.boot({ browser_id: webapp_client.browser_id }); } else if (data.cmd == "kill") { await this.api.kill(); } else { @@ -188,6 +187,13 @@ export class NatsTerminalConnection extends EventEmitter { } }; + kick = async () => { + await createBrowserClient({ + project_id: this.project_id, + path: this.path, + }).kick(webapp_client.browser_id); + }; + private createBrowserService = async () => { const impl = { command: async ({ event, paths }): Promise => { @@ -198,8 +204,11 @@ export class NatsTerminalConnection extends EventEmitter { } }, - kick: async ({ browser_id }) => { - console.log(`everyone but ${browser_id} must go!`); + kick: async (sender_browser_id) => { + console.log( + `everyone but ${sender_browser_id} must go!`, + webapp_client.browser_id, + ); }, size: async ({ rows, cols }) => { diff --git a/src/packages/nats/service/terminal.ts b/src/packages/nats/service/terminal.ts index 025031239e..0dde8ed5e0 100644 --- a/src/packages/nats/service/terminal.ts +++ b/src/packages/nats/service/terminal.ts @@ -30,9 +30,7 @@ export interface TerminalServiceApi { browser_id: string; }) => Promise; - boot: (opts: { browser_id: string }) => Promise; - - // send when this client is leaving. + // sent from browser to project when this client is leaving. close: (browser_id: string) => Promise; } @@ -68,8 +66,8 @@ export interface TerminalBrowserApi { // command is used for things like "open foo.txt" in the terminal. command: (mesg) => Promise; - // used for kicking user out of the terminal - kick: (opts: { browser_id: string }) => Promise; + // used for kicking all but the specified user out: + kick: (sender_browser_id: string) => Promise; // tell browser to change its size size: (opts: { rows: number; cols: number }) => Promise; diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index f8f126e76c..8938679299 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -89,10 +89,6 @@ export async function createTerminalService(path: string) { session.setSize(opts); }, - boot: async (opts: { browser_id: string }): Promise => { - console.log("boot", opts); - }, - close: async (browser_id: string) => { sessions[path]?.browserLeaving(browser_id); }, From 98b24b9b58a64e6d0262f778b8baf103c4d3d072 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 16 Feb 2025 05:01:32 +0000 Subject: [PATCH 212/281] nats services: incorporate stats/info/ping/waitFor --- .../terminal-editor/connected-terminal.ts | 3 +- .../nats-terminal-connection.ts | 11 ++- src/packages/nats/service/service.ts | 91 +++++++++++++++---- src/packages/nats/service/terminal.ts | 10 +- src/packages/nats/service/typed.ts | 34 ++++++- src/packages/server/nats/auth.ts | 10 +- src/packages/server/nats/index.ts | 2 +- 7 files changed, 129 insertions(+), 32 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 0b57d43e8b..5dae04561a 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -800,7 +800,8 @@ export class Terminal { } kick_other_users_out(): void { - this.conn.kick(); + // @ts-ignore + this.conn?.kick(); } kill(): void { diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index fbd7f40307..3ac2b12ea1 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -140,6 +140,7 @@ export class NatsTerminalConnection extends EventEmitter { }; private start = reuseInFlight(async () => { + await this.api.nats.waitFor(); await this.api.create(this.options); }); @@ -187,11 +188,15 @@ export class NatsTerminalConnection extends EventEmitter { } }; - kick = async () => { - await createBrowserClient({ + private browserClient = () => { + return createBrowserClient({ project_id: this.project_id, path: this.path, - }).kick(webapp_client.browser_id); + }); + }; + + kick = async () => { + await this.browserClient().kick(webapp_client.browser_id); }; private createBrowserService = async () => { diff --git a/src/packages/nats/service/service.ts b/src/packages/nats/service/service.ts index d1decc35b3..afb8cfc875 100644 --- a/src/packages/nats/service/service.ts +++ b/src/packages/nats/service/service.ts @@ -10,11 +10,17 @@ Also if the handler throws an error, the caller will throw an error too. */ -import { Svcm } from "@nats-io/services"; +import { + Svcm, + type ServiceInfo, + type ServiceStats, + type ServiceIdentity, +} from "@nats-io/services"; import { type NatsEnv } from "@cocalc/nats/types"; import { sha1, trunc_middle } from "@cocalc/util/misc"; import { getEnv } from "@cocalc/nats/client"; import { randomId } from "@cocalc/nats/names"; +import { delay } from "awaiting"; const DEFAULT_TIMEOUT = 5000; @@ -202,25 +208,74 @@ export class NatsService { }; } -/* -import { delay } from "awaiting"; -async function callWithRetry(f, maxTime) { +interface ServiceClientOpts { + options: ServiceDescription; + maxWait?: number; + id?: string; +} + +export async function pingNatsService({ + options, + maxWait = 500, + id, +}: ServiceClientOpts): Promise { + const env = await getEnv(); + const svc = new Svcm(env.nc); + const m = svc.client({ maxWait, strategy: "stall" }); + const v: ServiceIdentity[] = []; + for await (const ping of await m.ping(serviceName(options), id)) { + v.push(ping); + } + return v; +} + +export async function natsServiceInfo({ + options, + maxWait = 500, + id, +}: ServiceClientOpts): Promise { + const env = await getEnv(); + const svc = new Svcm(env.nc); + const m = svc.client({ maxWait, strategy: "stall" }); + const v: ServiceInfo[] = []; + for await (const info of await m.info(serviceName(options), id)) { + v.push(info); + } + return v; +} + +export async function natsServiceStats({ + options, + maxWait = 500, + id, +}: ServiceClientOpts): Promise { + const env = await getEnv(); + const svc = new Svcm(env.nc); + const m = svc.client({ maxWait, strategy: "stall" }); + const v: ServiceStats[] = []; + for await (const stats of await m.stats(serviceName(options), id)) { + v.push(stats); + } + return v; +} + +export async function waitForNatsService({ + options, + maxWait = 30000, +}: { + options: ServiceDescription; + maxWait?: number; +}) { let d = 100; + let m = 100; const start = Date.now(); - while (Date.now() - start < maxTime) { - try { - return await f(); - } catch (err) { - if (err.code == "503") { - d = Math.min(3000, d * 1.3); - if (Date.now() + d - start >= maxTime) { - throw err; - } - await delay(d); - continue; - } - throw err; + while ((await pingNatsService({ options, maxWait: m })).length == 0) { + d = Math.min(10000, d * 1.3); + m = Math.min(1500, m * 1.3); + console.log("waiting for terminal api to start...", d); + if (Date.now() - start + d >= maxWait) { + throw Error("timeout"); } + await delay(d); } } -*/ diff --git a/src/packages/nats/service/terminal.ts b/src/packages/nats/service/terminal.ts index 0dde8ed5e0..69e0f70bc6 100644 --- a/src/packages/nats/service/terminal.ts +++ b/src/packages/nats/service/terminal.ts @@ -8,7 +8,7 @@ export const SIZE_TIMEOUT_MS = 45000; // API that runs under Node.js in linux: -export interface TerminalServiceApi { +interface TerminalApi { create: (opts: { env?: { [key: string]: string }; command?: string; @@ -35,13 +35,15 @@ export interface TerminalServiceApi { } export function createTerminalClient({ project_id, path }) { - return createServiceClient({ + return createServiceClient({ project_id, path, service: "project-api", }); } +export type TerminalServiceApi = ReturnType; + export async function createTerminalServer({ project_id, path, @@ -49,9 +51,9 @@ export async function createTerminalServer({ }: { project_id: string; path: string; - impl: TerminalServiceApi; + impl; }) { - return await createServiceHandler({ + return await createServiceHandler({ project_id, path, service: "project-api", diff --git a/src/packages/nats/service/typed.ts b/src/packages/nats/service/typed.ts index 639ebd44a9..28b76fcf39 100644 --- a/src/packages/nats/service/typed.ts +++ b/src/packages/nats/service/typed.ts @@ -1,6 +1,24 @@ -import { callNatsService, createNatsService } from "./service"; +import { + callNatsService, + createNatsService, + natsServiceInfo, + natsServiceStats, + pingNatsService, + waitForNatsService, +} from "./service"; import type { Options, ServiceCall } from "./service"; +export interface Extra { + info: typeof natsServiceInfo; + stats: typeof natsServiceStats; + ping: typeof pingNatsService; + waitFor: (opts?: { maxWait?: number }) => Promise; +} + +export interface ServiceApi { + nats: Extra; +} + export function createServiceClient(options: Omit) { return new Proxy( {}, @@ -9,6 +27,18 @@ export function createServiceClient(options: Omit) { if (typeof prop !== "string") { return undefined; } + if (prop == "nats") { + return { + info: async (opts: { id?: string; maxWait?: number } = {}) => + await natsServiceInfo({ options, ...opts }), + stats: async (opts: { id?: string; maxWait?: number } = {}) => + await natsServiceStats({ options, ...opts }), + ping: async (opts: { id?: string; maxWait?: number } = {}) => + await pingNatsService({ options, ...opts }), + waitFor: async (opts: { maxWait?: number } = {}) => + await waitForNatsService({ options, ...opts }), + }; + } return async (...args) => { return await callNatsService({ ...options, @@ -17,7 +47,7 @@ export function createServiceClient(options: Omit) { }; }, }, - ) as Api; + ) as Api & ServiceApi; } export async function createServiceHandler({ diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index c8b260c9d5..65059ac137 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -124,10 +124,14 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { goalPub.add("$JS.API.*.*.public"); goalPub.add("$JS.API.*.*.public.>"); goalPub.add("$JS.API.CONSUMER.MSG.NEXT.public.>"); - - // microservices info api + + // microservices info api -- TODO: security concerns! + // Please don't tell me I have to name all microservice identically :-( goalSub.add(`$SRV.>`); - goalPub.add(`$SRV.*`); + goalPub.add(`$SRV.>`); + // TODO/security: just doing the following is enough if we don't need to use the client + // api to get stats/info about all services: + // goalPub.add(`$SRV.*`); if (userType == "account") { goalSub.add(`*.account-${userId}.>`); diff --git a/src/packages/server/nats/index.ts b/src/packages/server/nats/index.ts index 3ed28f115c..cffb42b33e 100644 --- a/src/packages/server/nats/index.ts +++ b/src/packages/server/nats/index.ts @@ -1,7 +1,7 @@ import getLogger from "@cocalc/backend/logger"; import { initAPI } from "./api"; import { init as initDatabase } from "@cocalc/database/nats/changefeeds"; -f + const logger = getLogger("server:nats"); export default async function initNatsServer() { From 7b215d6af3e3e4059fdc8d20bba75ea340e302e8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 16 Feb 2025 14:03:13 +0000 Subject: [PATCH 213/281] nats: improve terminal startup robustness --- .../terminal-editor/connected-terminal.ts | 6 +-- .../nats-terminal-connection.ts | 49 +++++++++++++------ src/packages/nats/service/service.ts | 7 ++- src/packages/project/nats/terminal.ts | 3 ++ src/packages/server/nats/auth.ts | 16 ++++-- 5 files changed, 57 insertions(+), 24 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 5dae04561a..6390032217 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -26,7 +26,6 @@ import { ProjectActions, redux } from "@cocalc/frontend/app-framework"; import { get_buffer, set_buffer } from "@cocalc/frontend/copy-paste-buffer"; import { file_associations } from "@cocalc/frontend/file-associations"; import { isCoCalcURL } from "@cocalc/frontend/lib/cocalc-urls"; -import type { Channel } from "@cocalc/comm/websocket/types"; import { aux_file, bind_methods, @@ -91,8 +90,7 @@ export class Terminal { private last_geom: { rows: number; cols: number } | undefined; private resize_after_no_ignore: { rows: number; cols: number } | undefined; private last_active: number = 0; - // conn = connection to project -- a primus websocket channel. - private conn?: Channel; + private conn?: NatsTerminalConnection; private touch_interval: any; // number doesn't work anymore and Timer doesn't exist everywhere... headache. Todo. public is_visible: boolean = false; @@ -816,7 +814,7 @@ export class Terminal { init_terminal_data(): void { this.terminal.onData((data) => { - if (this.ignore_terminal_data) { + if (this.ignore_terminal_data && this.conn?.state == "init") { return; } this.conn_write(data); diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 3ac2b12ea1..1249c16b60 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -12,12 +12,12 @@ import { } from "@cocalc/nats/service/terminal"; import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; -type State = "init" | "running" | "closed"; +type State = "disconnected" | "init" | "running" | "closed"; export class NatsTerminalConnection extends EventEmitter { private project_id: string; private path: string; - private state: State = "init"; + public state: State = "init"; private stream?: DStream; private terminalResize; private openPaths; @@ -56,14 +56,20 @@ export class NatsTerminalConnection extends EventEmitter { this.closePaths = closePaths; } + setState = (state: State) => { + this.state = state; + this.emit(state); + }; + write = async (data) => { - if (this.state == "init") { + if (this.state == "init" || this.state == "closed") { // ignore initial data while initializing. // This is the trickt to avoid "junk characters" on refresh/reconnect. return; } - if (this.state != "running") { - await this.start(); + if (this.state == "disconnected") { + await this.init(); + return; } if (typeof data != "string") { if (data.cmd == "size") { @@ -97,8 +103,6 @@ export class NatsTerminalConnection extends EventEmitter { this.api.write(data); } catch (err) { console.log(err); - // TODO: obviously wrong! A timeout would restart our poor terminal! - await this.start(); } }; @@ -132,7 +136,7 @@ export class NatsTerminalConnection extends EventEmitter { this.service?.close(); delete this.service; this.api.close(webapp_client.browser_id); - this.state = "closed"; + this.setState("closed"); }; end = () => { @@ -140,8 +144,17 @@ export class NatsTerminalConnection extends EventEmitter { }; private start = reuseInFlight(async () => { - await this.api.nats.waitFor(); - await this.api.create(this.options); + this.setState("init"); + try { + await this.api.nats.waitFor({ maxWait: 5000 }); + await this.api.create(this.options); + } catch (err) { + this.setState("disconnected"); + this.emit( + "data", + `\r\n\r\nUnable to start terminal - ${err}\r\n\r\n[Process not started - press any key]\r\n\r\n`, + ); + } }); private getStream = async () => { @@ -154,8 +167,16 @@ export class NatsTerminalConnection extends EventEmitter { }; init = async () => { - this.state = "init"; + this.setState("init"); await this.start(); + if (this.state == ("disconnected" as State)) { + // start failed + return; + } + if (this.stream != null) { + this.stream.close(); + delete this.stream; + } this.stream = await this.getStream(); this.consumeDataStream(); }; @@ -182,10 +203,8 @@ export class NatsTerminalConnection extends EventEmitter { // wait until after render loop of terminal before allowing writing, // or we get corruption. await delay(100); // todo is there a better way to know how long to wait? - if (this.state == "init") { - this.state = "running"; - this.emit("ready"); - } + this.setState("running"); + this.emit("ready"); }; private browserClient = () => { diff --git a/src/packages/nats/service/service.ts b/src/packages/nats/service/service.ts index afb8cfc875..2750a2964f 100644 --- a/src/packages/nats/service/service.ts +++ b/src/packages/nats/service/service.ts @@ -269,13 +269,16 @@ export async function waitForNatsService({ let d = 100; let m = 100; const start = Date.now(); - while ((await pingNatsService({ options, maxWait: m })).length == 0) { + let ping = await pingNatsService({ options, maxWait: m }); + while (ping.length == 0) { d = Math.min(10000, d * 1.3); m = Math.min(1500, m * 1.3); - console.log("waiting for terminal api to start...", d); if (Date.now() - start + d >= maxWait) { + console.log(`timeout waiting for ${serviceName(options)} to start...`, d); throw Error("timeout"); } await delay(d); + ping = await pingNatsService({ options, maxWait: m }); } + return ping; } diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index 8938679299..def76c2cca 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -195,6 +195,9 @@ class Session { write = async (data) => { if (this.state == "off") { await this.restart(); + // don't write when it starts it, since this is often a carriage return or space, + // which you don't want to send except to start it. + return; } this.pty?.write(data); }; diff --git a/src/packages/server/nats/auth.ts b/src/packages/server/nats/auth.ts index 65059ac137..fc6c33ded8 100644 --- a/src/packages/server/nats/auth.ts +++ b/src/packages/server/nats/auth.ts @@ -114,6 +114,10 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { const goalPub = new Set([ `hub.${userType}.${userId}.>`, // can talk as *only this user* to the hub's api's "$JS.API.INFO", + // everyone can publish to all inboxes. This seems like a major security risk, but with + // request/reply, the reply subject under _inbox is a long random code that is only known + // for a moment by the sender and the service, so I think it is NOT a security risk. + "_INBOX.>", ]); const goalSub = new Set([ inboxPrefix(cocalcUser) + ".>", @@ -203,14 +207,20 @@ export async function configureNatsUser(cocalcUser: CoCalcUser) { // We edit the signing key rather than the user, so the cookie in the user's // browser stays small and never has to change. - // Also,--allow-pub-response is explained at + // There is an option --allow-pub-response explained at // https://docs.nats.io/running-a-nats-service/configuration/securing_nats/authorization#allow-responses-map - // and makes it so we don't have to allow any user to publish to + // which is supposed to makes it so we don't have to allow any user to publish to // all of _INBOX.>, which might be bad, since one user could in theory // publish a response to a different user's request (though in practice // the subject is random so not feasible). Defense in depth. + // It doesn't work in general though, e.g., when trying to get info about services. - const args = ["edit", "signing-key", "--sk", name, "--allow-pub-response"]; + const args = [ + "edit", + "signing-key", + "--sk", + name /*, "--allow-pub-response" */, + ]; let changed = false; if (rm.length > 0) { From 16b8e0eb2e6c34c28930c5d694faef132f24927d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 16 Feb 2025 14:10:24 +0000 Subject: [PATCH 214/281] nats terminal: implement 'kick' --- .../frame-editors/terminal-editor/connected-terminal.ts | 5 +++-- .../terminal-editor/nats-terminal-connection.ts | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 6390032217..9f588885f2 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -292,6 +292,7 @@ export class Terminal { }); this.conn = conn as any; conn.on("close", this.connect); + conn.on("kick", this.close_request); conn.on("data", this._handle_data_from_project); conn.once("ready", () => { delete this.last_geom; @@ -682,7 +683,7 @@ export class Terminal { await callback(g); } - close_request(): void { + close_request = (): void => { this.actions.set_error("You were removed from a terminal."); // If there is only one frame, we close the // entire editor -- otherwise, we close only @@ -695,7 +696,7 @@ export class Terminal { } else { this.actions.close_frame(this.id); } - } + }; private use_subframe(path: string): boolean { const this_path_ext = filename_extension(this.actions.path); diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 1249c16b60..a39badde73 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -229,10 +229,11 @@ export class NatsTerminalConnection extends EventEmitter { }, kick: async (sender_browser_id) => { - console.log( - `everyone but ${sender_browser_id} must go!`, - webapp_client.browser_id, - ); + if (sender_browser_id == webapp_client.browser_id) { + // I sent the kick + return; + } + this.emit("kick"); }, size: async ({ rows, cols }) => { From c61fcb4330358dd8542821282072328690c232eb Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 16 Feb 2025 15:05:20 +0000 Subject: [PATCH 215/281] nats terminal: deleting old code, mostly --- .../terminal-editor/connected-terminal.ts | 306 +++++------------- .../nats-terminal-connection.ts | 9 +- .../terminal-editor/terminal.tsx | 10 +- .../project/page/flyouts/files-terminal.tsx | 2 +- src/packages/project/nats/terminal.ts | 17 +- 5 files changed, 97 insertions(+), 247 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 9f588885f2..f41f942034 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -6,13 +6,11 @@ /* Wrapper object around xterm.js's Terminal, which adds extra support for being connected to: - - a backend project via a websocket + - a backend project via NATS - react/redux - frame-editor (via actions) */ -const USE_NATS = true; - import { callback, delay } from "awaiting"; import { Map } from "immutable"; import { debounce } from "lodash"; @@ -28,7 +26,6 @@ import { file_associations } from "@cocalc/frontend/file-associations"; import { isCoCalcURL } from "@cocalc/frontend/lib/cocalc-urls"; import { aux_file, - bind_methods, close, endswith, filename_extension, @@ -36,7 +33,7 @@ import { } from "@cocalc/util/misc"; import { Actions, CodeEditorState } from "../code-editor/actions"; import { ConnectionStatus } from "../frame-tree/types"; -import { project_websocket, touch, touch_project } from "../generic/client"; +import { touch, touch_project } from "../generic/client"; import { ConnectedTerminalInterface } from "./connected-terminal-interface"; import { open_init_file } from "./init-file"; import { setTheme } from "./themes"; @@ -73,6 +70,7 @@ export class Terminal { readonly rendererType: "dom" | "canvas"; private terminal: XTerminal; private is_paused: boolean = false; + private pauseKeyCount: number = 0; private keyhandler_initialized: boolean = false; /* We initially have to ignore when rendering the initial history. To TEST this, do this in a terminal, then reconnect: @@ -91,7 +89,7 @@ export class Terminal { private resize_after_no_ignore: { rows: number; cols: number } | undefined; private last_active: number = 0; private conn?: NatsTerminalConnection; - private touch_interval: any; // number doesn't work anymore and Timer doesn't exist everywhere... headache. Todo. + private touch_interval; public is_visible: boolean = false; public element: HTMLElement; @@ -114,10 +112,9 @@ export class Terminal { args?: string[], workingDir?: string, ) { - bind_methods(this); + this.actions = actions; this.ask_for_cwd = debounce(this.ask_for_cwd); - this.actions = actions; this.account_store = redux.getStore("account"); this.project_actions = redux.getProjectActions(actions.project_id); if (this.account_store == null) { @@ -189,7 +186,7 @@ export class Terminal { // this.terminal_resize = debounce(this.terminal_resize, 2000); } - private get_xtermjs_options(): any { + private get_xtermjs_options = (): any => { const rendererType = this.rendererType; const settings = this.account_store.get("terminal"); if (settings == null) { @@ -211,13 +208,13 @@ export class Terminal { // const useFlowControl = true; return { rendererType, scrollback, fontFamily }; - } + }; - private assert_not_closed(): void { + private assert_not_closed = (): void => { if (this.state === "closed") { throw Error("BUG -- Terminal is closed."); } - } + }; close = (): void => { this.assert_not_closed(); @@ -233,7 +230,7 @@ export class Terminal { this.state = "closed"; }; - private disconnect(): void { + private disconnect = (): void => { if (this.conn === undefined) { return; } @@ -241,9 +238,9 @@ export class Terminal { this.conn.end(); delete this.conn; this.set_connection_status("disconnected"); - } + }; - private update_settings(): void { + private update_settings = (): void => { this.assert_not_closed(); const settings = this.account_store.get("terminal"); if (settings == null || this.terminal_settings.equals(settings)) { @@ -270,19 +267,21 @@ export class Terminal { } this.terminal_settings = settings; - } + }; - connectNats = async () => { + connect = async () => { + console.log("connect", this.state); try { this.ignore_terminal_data = true; this.set_connection_status("connecting"); + await this.configureComputeServerId(); const conn = new NatsTerminalConnection({ path: this.term_path, project_id: this.project_id, terminalResize: this.terminal_resize, openPaths: this.open_paths, closePaths: this.close_paths, - measureSize: this.measure_size, + measureSize: this.measureSize, options: { command: this.command, args: this.args, @@ -293,7 +292,10 @@ export class Terminal { this.conn = conn as any; conn.on("close", this.connect); conn.on("kick", this.close_request); - conn.on("data", this._handle_data_from_project); + conn.on("cwd", (cwd) => { + this.actions.set_terminal_cwd(this.id, cwd); + }); + conn.on("data", this.handleDataFromProject); conn.once("ready", () => { delete this.last_geom; this.ignore_terminal_data = false; @@ -301,8 +303,11 @@ export class Terminal { this.scroll_to_bottom(); this.terminal.refresh(0, this.terminal.rows - 1); this.init_keyhandler(); - this.measure_size(); + this.measureSize(); }); + if (endswith(this.path, ".term")) { + touchPath(this.project_id, this.path); // no need to await + } await conn.init(); } catch (err) { this.set_connection_status("disconnected"); @@ -310,117 +315,38 @@ export class Terminal { } }; - async connect(): Promise { - if (USE_NATS) { - return await this.connectNats(); - } - this.assert_not_closed(); - - this.last_geom = undefined; - if (this.conn != null) { - this.disconnect(); - } - try { - this.set_connection_status("connecting"); - - await this.configureComputeServerId(); - const ws = await project_websocket(this.project_id); - if (this.state === "closed") { - return; - } - const options: any = {}; - if (this.command != null) { - options.command = this.command; - } - if (this.args != null) { - options.args = this.args; - } - if (this.workingDir != null) { - options.workingDir = this.workingDir; - } - options.env = this.actions.get_term_env(); - options.path = this.path; - this.conn = await ws.api.terminal(this.term_path, options); - if (this.state === "closed") { - return; - } - } catch (err) { - if (this.state === "closed") { - return; - } - this.set_connection_status("disconnected"); - // console.log(`terminal connect error -- ${err}; will try again in 2s...`); - await delay(2000); - if (this.state === "closed") { - return; - } - this.connect(); - return; - } - if (this.conn == null) { - throw Error("bug"); - } - - // Delete any data or state in terminal before receiving new data. - this.terminal.reset(); - // Ignore device attr data coming back for initial load. - this.ignore_terminal_data = true; - this.conn.on("close", this.connect); - this.conn.on("data", this._handle_data_from_project); - if (endswith(this.path, ".term")) { - touch_path(this.project_id, this.path); // no need to await - } - for (const data of this.conn_write_buffer) { - this.conn.write(data); - } - this.conn_write_buffer = []; - this.set_connection_status("connected"); - this.ask_for_cwd(); - } - - async reload(): Promise { + reload = async (): Promise => { await this.connect(); - } + }; - conn_write(data): void { + conn_write = (data): void => { if (this.state == "closed") return; // no-op -- see #4918 if (this.conn === undefined) { this.conn_write_buffer.push(data); return; } this.conn.write(data); - } + }; - private _handle_data_from_project = (data: any): void => { + private handleDataFromProject = (data: any): void => { //console.log("data", data); this.assert_not_closed(); - if (data == null) { + if (!data || typeof data != "string") { return; } this.activity(); - switch (typeof data) { - case "string": - if (this.is_paused && !this.ignore_terminal_data) { - this.render_buffer += data; - } else { - this.render(data); - } - break; - - case "object": - this.handle_mesg(data); - break; - - default: - console.warn("TERMINAL: no way to handle data -- ", data); + if (this.is_paused && !this.ignore_terminal_data) { + this.render_buffer += data; + } else { + this.render(data); } }; - private activity() { + private activity = () => { this.project_actions.flag_file_activity(this.path); - } + }; - async render(data: string): Promise { + render = async (data: string): Promise => { this.assert_not_closed(); this.history += data; if (this.history.length > MAX_HISTORY_LENGTH) { @@ -437,29 +363,29 @@ export class Terminal { while (this.render_done.length > 0) { this.render_done.pop()?.(); } - } + }; // blocks until the next call to this.render - async wait_for_next_render(): Promise { + wait_for_next_render = async (): Promise => { return new Promise((done, _) => { this.render_done.push(done); }); - } + }; - init_title(): void { + init_title = (): void => { this.terminal.onTitleChange((title) => { if (title != null) { this.actions.set_title(this.id, title); this.ask_for_cwd(); } }); - } + }; - set_connection_status(status: ConnectionStatus): void { + set_connection_status = (status: ConnectionStatus): void => { if (this.actions != null) { this.actions.set_connection_status(this.id, status); } - } + }; touch = async () => { if (Date.now() - this.last_active < 70000) { @@ -467,11 +393,11 @@ export class Terminal { } }; - init_touch(): void { + init_touch = (): void => { this.touch_interval = setInterval(this.touch, 60000); - } + }; - init_keyhandler(): void { + init_keyhandler = (): void => { if (this.state === "closed") { return; } @@ -497,7 +423,13 @@ export class Terminal { this.ignore_terminal_data = false; if (this.is_paused) { - this.actions.unpause(this.id); + this.pauseKeyCount += 1; + if (this.pauseKeyCount >= 4) { + // otherwise, trying to copy when paused causes it to unpause which is + // very annoying. there's a button... but if the user forgets and starts + // mashing buttons, it still works. + this.actions.unpause(this.id); + } } if ( @@ -535,83 +467,7 @@ export class Terminal { return true; }); - } - - handle_mesg(mesg: { - cmd: string; - rows?: number; - cols?: number; - payload: any; - ignore?: string; - id?: number; - }): void { - //console.log("handle_mesg", this.id, mesg); - switch (mesg.cmd) { - case "size": - if (typeof mesg.rows == "number" && typeof mesg.cols == "number") { - this.terminal_resize({ rows: mesg.rows, cols: mesg.cols }); - } - break; - case "cwd": - this.actions.set_terminal_cwd(this.id, mesg.payload); - break; - case "burst": - this.burst_on(); - break; - case "no-burst": - this.burst_off(); - break; - case "no-ignore": - this.no_ignore(); - break; - case "close": - if (mesg.ignore != this.id) { - this.close_request(); - } - break; - case "computeServerId": - if (this.actions.store != null && this.actions.setState != null) { - const terminalComputeServerIds = - this.actions.store.get("terminalComputeServerIds")?.toJS() ?? {}; - terminalComputeServerIds[this.term_path] = mesg.id; - this.actions.setState({ terminalComputeServerIds }); - } - break; - case "message": - const payload = mesg.payload; - if (payload == null) { - break; - } - switch (payload.event) { - case "open": - if (payload.paths !== undefined) { - this.open_paths(payload.paths); - } - break; - case "close": - if (payload.paths !== undefined) { - this.close_paths(payload.paths); - } - break; - } - - break; - default: - console.warn("handle_mesg -- unhandled", this.id, mesg); - } - } - - burst_on(): void { - // TODO: would be better to make specific to that terminal... but not implemented. - const mesg = "WARNING: Large burst of output! (May try to interrupt.)"; - this.actions.set_status(mesg); - this.actions.set_error(mesg); - } - - burst_off(): void { - this.actions.set_status(""); - this.actions.set_error(""); - } + }; // Try to resize terminal to given number of rows and columns. // This should not throw an exception no matter how wrong the input @@ -652,7 +508,7 @@ export class Terminal { // Stop ignoring terminal data... but ONLY once // the render buffer is also empty. - async no_ignore(): Promise { + no_ignore = async (): Promise => { if (this.state === "closed") { return; } @@ -681,7 +537,7 @@ export class Terminal { const x = this.terminal.onRender(f); }; await callback(g); - } + }; close_request = (): void => { this.actions.set_error("You were removed from a terminal."); @@ -698,7 +554,7 @@ export class Terminal { } }; - private use_subframe(path: string): boolean { + private use_subframe = (path: string): boolean => { const this_path_ext = filename_extension(this.actions.path); if (this_path_ext == "term") { // This is a .term tab, so always open the path in a new editor @@ -717,7 +573,7 @@ export class Terminal { return true; } return false; - } + }; private open_paths = async (paths: Path[]) => { if (!this.is_visible) { @@ -752,10 +608,10 @@ export class Terminal { } }; - _close_path(path: string): void { + _close_path = (path: string): void => { const project_actions = this.actions._get_project_actions(); project_actions.close_tab(path); - } + }; close_paths = (paths: Path[]): void => { if (!this.is_visible) { @@ -768,7 +624,7 @@ export class Terminal { } }; - resize(rows: number, cols: number): void { + resize = (rows: number, cols: number): void => { if (this.terminal.cols === cols && this.terminal.rows === rows) { // no need to resize return; @@ -782,21 +638,22 @@ export class Terminal { return; } this.terminal_resize({ rows, cols }); - } + }; - pause(): void { + pause = (): void => { this.is_paused = true; - } + this.pauseKeyCount = 0; + }; - unpause(): void { + unpause = (): void => { this.is_paused = false; this.render(this.render_buffer); this.render_buffer = ""; - } + }; - ask_for_cwd(): void { + ask_for_cwd = (): void => { this.conn_write({ cmd: "cwd" }); - } + }; kick_other_users_out(): void { // @ts-ignore @@ -862,17 +719,12 @@ export class Terminal { return this.terminal.options[option]; } - measure_size = (): void => { + measureSize = (): void => { if (this.ignore_terminal_data) { // during initial load return; } const geom = this.fitAddon.proposeDimensions(); - // console.log("measure_size", { - // geom, - // ignore: this.ignore_terminal_data, - // last_geom: this.last_geom, - // }); if (geom == null) { return; } @@ -884,25 +736,25 @@ export class Terminal { this.conn_write({ cmd: "size", rows, cols }); }; - copy(): void { + copy = (): void => { const sel: string = this.terminal.getSelection(); set_buffer(sel); this.terminal.focus(); - } + }; - paste(): void { + paste = (): void => { this.terminal.clearSelection(); this.terminal.paste(get_buffer()); this.terminal.focus(); - } + }; - scroll_to_bottom(): void { + scroll_to_bottom = (): void => { // Upstream bug workaround -- we scroll to top first, then bottom // entirely to workaround a bug. This is NOT fixed by the Oct 2018 // term.js release, despite it touching relevant code. this.terminal.scrollToTop(); this.terminal.scrollToBottom(); - } + }; getComputeServerId = async (): Promise => { const computeServerAssociations = @@ -949,7 +801,7 @@ export class Terminal { }; } -async function touch_path(project_id: string, path: string): Promise { +async function touchPath(project_id: string, path: string): Promise { // touch the original path file on disk, so it exists and is // modified -- that's the ONLY purpose of this touch. // Also this is in a separate function so we can await it and catch exception. diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index a39badde73..8b2f0e719d 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -181,8 +181,7 @@ export class NatsTerminalConnection extends EventEmitter { this.consumeDataStream(); }; - private handleStreamMessage = (mesg) => { - const data = mesg?.data; + private handleStreamData = (data) => { if (data) { this.emit("data", data); } @@ -192,11 +191,11 @@ export class NatsTerminalConnection extends EventEmitter { if (this.stream == null) { return; } - for (const mesg of this.stream.get()) { - this.handleStreamMessage(mesg); + for (const data of this.stream.get()) { + this.handleStreamData(data); } this.setReady(); - this.stream.on("change", this.handleStreamMessage); + this.stream.on("change", this.handleStreamData); }; private setReady = async () => { diff --git a/src/packages/frontend/frame-editors/terminal-editor/terminal.tsx b/src/packages/frontend/frame-editors/terminal-editor/terminal.tsx index 75d6af4876..478c3964e7 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/terminal.tsx +++ b/src/packages/frontend/frame-editors/terminal-editor/terminal.tsx @@ -92,7 +92,7 @@ export const TerminalFrame: React.FC = React.memo((props: Props) => { }, [props.is_current]); useEffect(() => { - measure_size(); + measureSize(); }, [props.resize, resize]); function delete_terminal(): void { @@ -119,7 +119,7 @@ export const TerminalFrame: React.FC = React.memo((props: Props) => { if (terminalRef.current == null) return; // should be impossible. terminalRef.current.is_visible = true; set_font_size(); - measure_size(); + measureSize(); if (props.is_current) { terminalRef.current.focus(); } @@ -141,15 +141,15 @@ export const TerminalFrame: React.FC = React.memo((props: Props) => { } if (terminalRef.current.getOption("fontSize") !== props.font_size) { terminalRef.current.set_font_size(props.font_size); - measure_size(); + measureSize(); } }, 200); useEffect(set_font_size, [props.font_size]); - function measure_size(): void { + function measureSize(): void { if (isMountedRef.current) { - terminalRef.current?.measure_size(); + terminalRef.current?.measureSize(); } } diff --git a/src/packages/frontend/project/page/flyouts/files-terminal.tsx b/src/packages/frontend/project/page/flyouts/files-terminal.tsx index 8a62821ac9..c3f467dcde 100644 --- a/src/packages/frontend/project/page/flyouts/files-terminal.tsx +++ b/src/packages/frontend/project/page/flyouts/files-terminal.tsx @@ -275,7 +275,7 @@ export function TerminalFlyout({ function measure_size(): void { if (isMountedRef.current) { - terminalRef.current?.measure_size(); + terminalRef.current?.measureSize(); } } diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index def76c2cca..76e1a43f6e 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -278,11 +278,10 @@ class Session { logger.debug("connect stream to pty"); this.pty.onData((data) => { this.handleBackendMessages(data); - this.stream?.publish({ data }); + this.stream?.publish(data); }); - this.pty.onExit((status) => { - this.stream?.publish({ data: EXIT_MESSAGE }); - this.stream?.publish({ ...status, exit: true }); + this.pty.onExit(() => { + this.stream?.publish(EXIT_MESSAGE); this.state = "off"; }); }; @@ -305,7 +304,7 @@ class Session { this.resize(); }; - private resize = () => { + private resize = async () => { if (this.pty == null) { // nothing to do return; @@ -319,19 +318,19 @@ class Session { try { this.setSizePty({ rows, cols }); // tell browsers about out new size - this.browserApi.size({ rows, cols }); + await this.browserApi.size({ rows, cols }); } catch (err) { logger.debug("terminal channel -- WARNING: unable to resize term", err); } }; setSizePty = ({ rows, cols }: { rows: number; cols: number }) => { - logger.debug("setSize", { rows, cols }); + // logger.debug("setSize", { rows, cols }); if (this.pty == null) { - logger.debug("setSize: not doing since pty not defined"); + // logger.debug("setSize: not doing since pty not defined"); return; } - logger.debug("setSize", { rows, cols }, "DOING IT!"); + // logger.debug("setSize", { rows, cols }, "DOING IT!"); this.pty.resize(cols, rows); this.size = { rows, cols }; From b636af30baad0a4b3fbdc1284f75360bc7fb1ba0 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 16 Feb 2025 15:56:01 +0000 Subject: [PATCH 216/281] nats: make initial load of stream much faster and always load it all - similar to kv - now works properly :-) --- .../terminal-editor/connected-terminal.ts | 4 +- .../nats-terminal-connection.ts | 7 ++-- src/packages/nats/sync/stream.ts | 40 +++++++++++-------- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index f41f942034..b08467332e 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -270,11 +270,13 @@ export class Terminal { }; connect = async () => { - console.log("connect", this.state); try { this.ignore_terminal_data = true; this.set_connection_status("connecting"); await this.configureComputeServerId(); + if (this.state == "closed") { + return; + } const conn = new NatsTerminalConnection({ path: this.term_path, project_id: this.project_id, diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 8b2f0e719d..02eaa146f6 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -191,9 +191,8 @@ export class NatsTerminalConnection extends EventEmitter { if (this.stream == null) { return; } - for (const data of this.stream.get()) { - this.handleStreamData(data); - } + const initData = this.stream.get().join(""); + this.handleStreamData(initData); this.setReady(); this.stream.on("change", this.handleStreamData); }; @@ -201,7 +200,7 @@ export class NatsTerminalConnection extends EventEmitter { private setReady = async () => { // wait until after render loop of terminal before allowing writing, // or we get corruption. - await delay(100); // todo is there a better way to know how long to wait? + await delay(1); // todo is there a better way to know how long to wait? this.setState("running"); this.emit("ready"); }; diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 076911265d..12e0249899 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -310,27 +310,33 @@ export class Stream extends EventEmitter { private fetchInitialData = async (options?) => { const consumer = await this.getConsumer(options); - // This goes in two stages: - // STAGE 1: Get what is in the stream now. - // First we get info so we know how many messages - // are already in the stream: - const info = await consumer.info(); - if (info.num_pending == 0) { - return consumer; - } - const fetch = await consumer.fetch(); - this.watch = fetch; - let i = 0; // grab the messages. This should be very efficient since it // internally grabs them in batches. - for await (const mesg of fetch) { - this.handle(mesg, true); - i += 1; - if (i >= info.num_pending) { - break; + // This code seems exactly necessary and efficient, and most + // other things I tried ended too soon or hung. See also + // comment in getAllFromKv about permissions. + // const start = Date.now(); + // let count = 0; + //try { + while (true) { + const info = await consumer.info(); + if (info.num_pending == 0) { + return consumer; + } + const fetch = await consumer.fetch({ max_messages: 1000 }); + this.watch = fetch; + for await (const mesg of fetch) { + // count += 1; + const pending = mesg.info.pending; + this.handle(mesg, true); + if (pending <= 0) { + return consumer; + } } } - return consumer; + // } finally { + // console.log("fetchInitialData", { count, time: Date.now() - start }); + // } }; private watchForNewData = async (consumer) => { From 9124f53bd38a57c9e1646bfc44b08a037fa7ed16 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 16 Feb 2025 17:52:32 +0000 Subject: [PATCH 217/281] dstream -- support optional typing --- src/packages/backend/nats/sync.ts | 4 +- src/packages/backend/nats/test/dkv.test.ts | 3 ++ .../backend/nats/test/dstream.test.ts | 15 ++++++ .../nats-terminal-connection.ts | 7 ++- src/packages/frontend/nats/client.ts | 4 +- src/packages/nats/sync/dstream.ts | 54 ++++++++++++------- src/packages/project/nats/sync.ts | 4 +- src/packages/project/nats/terminal.ts | 38 ++----------- 8 files changed, 64 insertions(+), 65 deletions(-) diff --git a/src/packages/backend/nats/sync.ts b/src/packages/backend/nats/sync.ts index b698ea0a74..9f604620c3 100644 --- a/src/packages/backend/nats/sync.ts +++ b/src/packages/backend/nats/sync.ts @@ -15,8 +15,8 @@ export async function stream(opts): Promise { return await createStream({ env: await getEnv(), ...opts }); } -export async function dstream(opts): Promise { - return await createDstream({ env: await getEnv(), ...opts }); +export async function dstream(opts): Promise> { + return await createDstream({ env: await getEnv(), ...opts }); } export async function kv(opts): Promise { diff --git a/src/packages/backend/nats/test/dkv.test.ts b/src/packages/backend/nats/test/dkv.test.ts index 2c2f9d868b..7df50cb1ca 100644 --- a/src/packages/backend/nats/test/dkv.test.ts +++ b/src/packages/backend/nats/test/dkv.test.ts @@ -351,3 +351,6 @@ describe("tests involving null/undefined values", () => { expect(kv1.get()).toEqual({}); }); }); + + + diff --git a/src/packages/backend/nats/test/dstream.test.ts b/src/packages/backend/nats/test/dstream.test.ts index 17a442217d..6822d82a7f 100644 --- a/src/packages/backend/nats/test/dstream.test.ts +++ b/src/packages/backend/nats/test/dstream.test.ts @@ -236,3 +236,18 @@ describe("a little bit of a stress test", () => { expect(s.length).toBe(count); }); }); + +describe("dstream typescript test", () => { + it("creates stream", async () => { + const name = `test-${Math.random()}`; + const s = await createDstream({ name }); + + // write a message with the correct type + s.push("foo"); + + // wrong type -- no way to test this, but if you uncomment + // this you should get a typescript error: + + // s.push({ foo: "bar" }); + }); +}); diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 02eaa146f6..5926230db9 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -18,7 +18,7 @@ export class NatsTerminalConnection extends EventEmitter { private project_id: string; private path: string; public state: State = "init"; - private stream?: DStream; + private stream?: DStream; private terminalResize; private openPaths; private closePaths; @@ -158,9 +158,8 @@ export class NatsTerminalConnection extends EventEmitter { }); private getStream = async () => { - // TODO: idempotent, but move to project const { nats_client } = webapp_client; - return await nats_client.dstream({ + return await nats_client.dstream({ name: `terminal-${this.path}`, project_id: this.project_id, }); @@ -191,7 +190,7 @@ export class NatsTerminalConnection extends EventEmitter { if (this.stream == null) { return; } - const initData = this.stream.get().join(""); + const initData = this.stream.getAll().join(""); this.handleStreamData(initData); this.setReady(); this.stream.on("change", this.handleStreamData); diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 71204a3740..2eb6c47cc9 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -393,11 +393,11 @@ export class NatsClient { return await stream({ env: await this.getEnv(), ...opts }); }; - dstream = async (opts: Partial & { name: string }) => { + dstream = async (opts: Partial & { name: string }) => { if (!opts.account_id && !opts.project_id && opts.limits != null) { throw Error("account client can't set limits on public stream"); } - return await dstream({ env: await this.getEnv(), ...opts }); + return await dstream({ env: await this.getEnv(), ...opts }); }; kv = async (opts: Partial & { name: string }) => { diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 505e19fed2..215346284c 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -32,6 +32,7 @@ import { isNumericString } from "@cocalc/util/misc"; import { sha1 } from "@cocalc/util/misc"; import { millis } from "@cocalc/nats/util"; import refCache from "@cocalc/util/refcache"; +import { type JsMsg } from "@nats-io/jetstream"; const MAX_PARALLEL = 250; @@ -39,15 +40,15 @@ export interface DStreamOptions extends StreamOptions { noAutosave?: boolean; } -export class DStream extends EventEmitter { +export class DStream extends EventEmitter { public readonly name: string; private stream?: Stream; - private messages: any[]; - private raw: any[]; + private messages: T[]; + private raw: JsMsg[]; private noAutosave: boolean; // TODO: using Map for these will be better because we use .length a bunch, which is O(n) instead of O(1). - private local: { [id: string]: { mesg: any; subject?: string } } = {}; - private saved: { [seq: number]: any } = {}; + private local: { [id: string]: { mesg: T; subject?: string } } = {}; + private saved: { [seq: number]: T } = {}; constructor(opts: DStreamOptions) { super(); @@ -96,31 +97,38 @@ export class DStream extends EventEmitter { delete this.raw; }; - get = (n?) => { + get = (n?): T | T[] => { if (this.stream == null) { throw Error("closed"); } if (n == null) { - return [ - ...this.messages, - ...Object.values(this.saved), - ...Object.values(this.local).map((x) => x.mesg), - ]; + return this.getAll(); } else { if (n < this.messages.length) { return this.messages[n]; } const v = Object.keys(this.saved); if (n < v.length + this.messages.length) { - return v[n - this.messages.length]; + return this.saved[n - this.messages.length]; } return Object.values(this.local)[n - this.messages.length - v.length] ?.mesg; } }; + getAll = (): T[] => { + if (this.stream == null) { + throw Error("closed"); + } + return [ + ...this.messages, + ...Object.values(this.saved), + ...Object.values(this.local).map((x) => x.mesg), + ]; + }; + // sequence number of n-th message - seq = (n) => { + seq = (n: number): number | undefined => { if (n < this.raw.length) { return this.raw[n]?.seq; } @@ -130,7 +138,7 @@ export class DStream extends EventEmitter { } }; - time = (n) => { + time = (n: number): Date | undefined => { const r = this.raw[n]; if (r == null) { return; @@ -138,7 +146,7 @@ export class DStream extends EventEmitter { return new Date(millis(r?.info.timestampNanos)); }; - get length() { + get length(): number { return ( this.messages.length + Object.keys(this.saved).length + @@ -146,7 +154,7 @@ export class DStream extends EventEmitter { ); } - publish = (mesg, subject?: string) => { + publish = (mesg: T, subject?: string): void => { const id = randomId(); this.local[id] = { mesg, subject }; if (!this.noAutosave) { @@ -154,7 +162,7 @@ export class DStream extends EventEmitter { } }; - push = (...args) => { + push = (...args: T[]) => { if (this.stream == null) { throw Error("closed"); } @@ -163,14 +171,14 @@ export class DStream extends EventEmitter { } }; - hasUnsavedChanges = () => { + hasUnsavedChanges = (): boolean => { if (this.stream == null) { return false; } return Object.keys(this.local).length > 0; }; - unsavedChanges = () => { + unsavedChanges = (): T[] => { return Object.values(this.local).map(({ mesg }) => mesg); }; @@ -231,7 +239,7 @@ export class DStream extends EventEmitter { }; } -export const dstream = refCache({ +const cache = refCache({ createKey: userStreamOptionsKey, createObject: async (options) => { const { account_id, project_id, name } = options; @@ -250,3 +258,9 @@ export const dstream = refCache({ return dstream; }, }); + +export async function dstream( + options: UserStreamOptions, +): Promise> { + return await cache(options); +} diff --git a/src/packages/project/nats/sync.ts b/src/packages/project/nats/sync.ts index 9474ce2e11..df4845078f 100644 --- a/src/packages/project/nats/sync.ts +++ b/src/packages/project/nats/sync.ts @@ -20,8 +20,8 @@ export async function stream(opts): Promise { return await createStream({ project_id, env: await getEnv(), ...opts }); } -export async function dstream(opts): Promise { - return await createDstream({ project_id, env: await getEnv(), ...opts }); +export async function dstream(opts): Promise> { + return await createDstream({ project_id, env: await getEnv(), ...opts }); } export async function kv(opts): Promise { diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index 76e1a43f6e..a4dce047d6 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -139,38 +139,6 @@ export const createTerminal = reuseInFlight( }, ); -export async function writeToTerminal({ data, path }: { data; path }) { - const terminal = sessions[path]; - if (terminal == null) { - throw Error(`no terminal session '${path}'`); - } - await terminal.write(data); -} - -export async function restartTerminal({ path }: { path }) { - const terminal = sessions[path]; - if (terminal == null) { - throw Error(`no terminal session '${path}'`); - } - await terminal.restart(); -} - -export async function terminalCommand({ path, cmd, ...args }) { - logger.debug("terminalCommand", { path, cmd, args }); - const terminal = sessions[path]; - if (terminal == null) { - throw Error(`no terminal session '${path}'`); - } - switch (cmd) { - case "size": - return terminal.setSize(args as any); - case "cwd": - return await terminal.getCwd(); - default: - throw Error(`unknown cmd="${cmd}"`); - } -} - class Session { public state: "running" | "off" | "closed" = "off"; public options; @@ -178,7 +146,7 @@ class Session { private pty?; private size?: { rows: number; cols: number }; private browserApi: ReturnType; - private stream?: DStream; + private stream?: DStream; private streamName: string; private clientSizes: { [browser_id: string]: { rows: number; cols: number; time: number }; @@ -243,7 +211,7 @@ class Session { }; createStream = async () => { - this.stream = await dstream({ + this.stream = await dstream({ name: this.streamName, limits: { max_bytes: HISTORY_LIMIT_BYTES }, }); @@ -276,7 +244,7 @@ class Session { logger.debug("creating stream"); await this.createStream(); logger.debug("connect stream to pty"); - this.pty.onData((data) => { + this.pty.onData((data: string) => { this.handleBackendMessages(data); this.stream?.publish(data); }); From 586dfc3ba06eab7507b1ecb2be9795fda7ad86bf Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 16 Feb 2025 18:47:26 +0000 Subject: [PATCH 218/281] add typing support to DKO, DKV, KV, DStream, Stream --- src/packages/backend/nats/sync.ts | 12 +-- src/packages/backend/nats/test/dko.test.ts | 14 ++-- src/packages/backend/nats/test/dkv.test.ts | 53 ++++++------ .../backend/nats/test/dstream.test.ts | 24 +++--- .../backend/nats/test/open-files.test.ts | 7 +- .../backend/nats/test/service.test.ts | 2 +- src/packages/frontend/nats/client.ts | 42 ++++++---- src/packages/nats/sync/dko.ts | 80 +++++++++++-------- src/packages/nats/sync/dkv.ts | 59 ++++++++------ src/packages/nats/sync/general-dkv.ts | 44 +++++----- src/packages/nats/sync/general-kv.ts | 59 +++++++------- src/packages/nats/sync/kv.ts | 50 +++++++----- src/packages/nats/sync/open-files.ts | 6 +- src/packages/nats/sync/stream.ts | 42 ++++++---- src/packages/nats/sync/synctable-kv.ts | 4 +- src/packages/nats/sync/synctable-stream.ts | 2 +- src/packages/project/nats/sync.ts | 18 ++--- 17 files changed, 294 insertions(+), 224 deletions(-) diff --git a/src/packages/backend/nats/sync.ts b/src/packages/backend/nats/sync.ts index 9f604620c3..ef0d9aac36 100644 --- a/src/packages/backend/nats/sync.ts +++ b/src/packages/backend/nats/sync.ts @@ -11,23 +11,23 @@ import { createOpenFiles, type OpenFiles } from "@cocalc/nats/sync/open-files"; export type { Stream, DStream, KV, DKV, DKO }; -export async function stream(opts): Promise { - return await createStream({ env: await getEnv(), ...opts }); +export async function stream(opts): Promise> { + return await createStream({ env: await getEnv(), ...opts }); } export async function dstream(opts): Promise> { return await createDstream({ env: await getEnv(), ...opts }); } -export async function kv(opts): Promise { +export async function kv(opts): Promise> { return await createKV({ env: await getEnv(), ...opts }); } -export async function dkv(opts): Promise { - return await createDKV({ env: await getEnv(), ...opts }); +export async function dkv(opts): Promise> { + return await createDKV({ env: await getEnv(), ...opts }); } -export async function dko(opts): Promise { +export async function dko(opts): Promise> { return await createDKO({ env: await getEnv(), ...opts }); } diff --git a/src/packages/backend/nats/test/dko.test.ts b/src/packages/backend/nats/test/dko.test.ts index daa3e8c511..4a5a414330 100644 --- a/src/packages/backend/nats/test/dko.test.ts +++ b/src/packages/backend/nats/test/dko.test.ts @@ -16,7 +16,7 @@ describe("create a public kv and do basic operations", () => { it("creates the kv", async () => { kv = await createDko({ name }); - expect(kv.get()).toEqual({}); + expect(kv.getAll()).toEqual({}); }); it("adds a key to the kv", () => { @@ -39,7 +39,7 @@ describe("create a public kv and do basic operations", () => { it("closes the kv", async () => { kv.close(); - expect(kv.get).toThrow("closed"); + expect(kv.getAll).toThrow("closed"); }); }); @@ -51,19 +51,19 @@ describe("opens a kv twice and verifies the cached works and is reference counte it("creates the same kv twice", async () => { kv1 = await createDko({ name }); kv2 = await createDko({ name }); - expect(kv1.get()).toEqual({}); + expect(kv1.getAll()).toEqual({}); expect(kv1 === kv2).toBe(true); }); it("closes kv1 (one reference)", async () => { kv1.close(); - expect(kv2.get).not.toThrow(); + expect(kv2.getAll).not.toThrow(); }); it("closes kv2 (another reference)", async () => { kv2.close(); // really closed! - expect(kv2.get).toThrow("closed"); + expect(kv2.getAll).toThrow("closed"); }); it("create and see it is new now", async () => { @@ -80,8 +80,8 @@ describe("opens a kv twice at once and observe sync", () => { it("creates the kv twice", async () => { kv1 = await createDko({ name, noCache: true }); kv2 = await createDko({ name, noCache: true }); - expect(kv1.get()).toEqual({}); - expect(kv2.get()).toEqual({}); + expect(kv1.getAll()).toEqual({}); + expect(kv2.getAll()).toEqual({}); expect(kv1 === kv2).toBe(false); }); diff --git a/src/packages/backend/nats/test/dkv.test.ts b/src/packages/backend/nats/test/dkv.test.ts index 7df50cb1ca..eca64ad709 100644 --- a/src/packages/backend/nats/test/dkv.test.ts +++ b/src/packages/backend/nats/test/dkv.test.ts @@ -17,7 +17,7 @@ describe("create a public dkv and do basic operations", () => { it("creates the dkv", async () => { kv = await createDkv({ name }); - expect(kv.get()).toEqual({}); + expect(kv.getAll()).toEqual({}); }); it("adds a key to the dkv", () => { @@ -34,11 +34,11 @@ describe("create a public dkv and do basic operations", () => { it("closes the kv", async () => { kv.close(); - expect(kv.get).toThrow("closed"); + expect(kv.getAll).toThrow("closed"); }); }); -describe("opens a dkv twice and verifies the cached works and is reference counted", () => { +describe("opens a dkv twice and verifies the cache works and is reference counted", () => { let kv1; let kv2; const name = `test-${Math.random()}`; @@ -46,19 +46,19 @@ describe("opens a dkv twice and verifies the cached works and is reference count it("creates the same dkv twice", async () => { kv1 = await createDkv({ name }); kv2 = await createDkv({ name }); - expect(kv1.get()).toEqual({}); + expect(kv1.getAll()).toEqual({}); expect(kv1 === kv2).toBe(true); }); it("closes kv1 (one reference)", async () => { kv1.close(); - expect(kv2.get).not.toThrow(); + expect(kv2.getAll).not.toThrow(); }); it("closes kv2 (another reference)", async () => { kv2.close(); // really closed! - expect(kv2.get).toThrow("closed"); + expect(kv2.getAll).toThrow("closed"); }); it("create and see it is new now", async () => { @@ -75,8 +75,8 @@ describe("opens a dkv twice at once and observe sync", () => { it("creates the dkv twice", async () => { kv1 = await createDkv({ name, noCache: true }); kv2 = await createDkv({ name, noCache: true }); - expect(kv1.get()).toEqual({}); - expect(kv2.get()).toEqual({}); + expect(kv1.getAll()).toEqual({}); + expect(kv2.getAll()).toEqual({}); expect(kv1 === kv2).toBe(false); }); @@ -105,7 +105,7 @@ describe("check server assigned times", () => { it("create a kv", async () => { kv = await createDkv({ name }); - expect(kv.get()).toEqual({}); + expect(kv.getAll()).toEqual({}); expect(kv.time()).toEqual({}); }); @@ -147,8 +147,8 @@ describe("test deleting and clearing a dkv", () => { it("creates the dkv twice without caching so can make sure sync works", async () => { await reset(); - expect(kv1.get()).toEqual({}); - expect(kv2.get()).toEqual({}); + expect(kv1.getAll()).toEqual({}); + expect(kv2.getAll()).toEqual({}); expect(kv1 === kv2).toBe(false); }); @@ -217,7 +217,7 @@ describe("set several items, confirm write worked, save, and confirm they are st // the time thresholds should be "trivial" it(`adds ${count} entries`, async () => { const kv = await createDkv({ name }); - expect(kv.get()).toEqual({}); + expect(kv.getAll()).toEqual({}); const obj: any = {}; const t0 = Date.now(); for (let i = 0; i < count; i++) { @@ -225,11 +225,11 @@ describe("set several items, confirm write worked, save, and confirm they are st kv.set(`${i}`, i); } expect(Date.now() - t0).toBeLessThan(50); - expect(Object.keys(kv.get()).length).toEqual(count); - expect(kv.get()).toEqual(obj); + expect(Object.keys(kv.getAll()).length).toEqual(count); + expect(kv.getAll()).toEqual(obj); await kv.save(); expect(Date.now() - t0).toBeLessThan(1000); - expect(Object.keys(kv.get()).length).toEqual(count); + expect(Object.keys(kv.getAll()).length).toEqual(count); // // the local state maps should also get cleared quickly, // // but there is no event for this, so we loop: // @ts-ignore: saved is private @@ -248,17 +248,17 @@ describe("do an insert and clear test", () => { const count = 100; it(`adds ${count} entries, saves, clears, and confirms empty`, async () => { const kv = await createDkv({ name }); - expect(kv.get()).toEqual({}); + expect(kv.getAll()).toEqual({}); for (let i = 0; i < count; i++) { kv[`${i}`] = i; } - expect(Object.keys(kv.get()).length).toEqual(count); + expect(Object.keys(kv.getAll()).length).toEqual(count); await kv.save(); - expect(Object.keys(kv.get()).length).toEqual(count); + expect(Object.keys(kv.getAll()).length).toEqual(count); kv.clear(); - expect(kv.get()).toEqual({}); + expect(kv.getAll()).toEqual({}); await kv.save(); - expect(kv.get()).toEqual({}); + expect(kv.getAll()).toEqual({}); }); }); @@ -269,7 +269,7 @@ describe("create many distinct clients at once, write to all of them, and see th it(`creates the ${count} clients`, async () => { for (let i = 0; i < count; i++) { - clients[i] = await createDkv({ name, noCache: true }); + clients[i] = await createDkv({ name, noCache: true }); } }); @@ -302,7 +302,7 @@ describe("create many distinct clients at once, write to all of them, and see th } for (const client of clients) { expect(client.length).toEqual(count); - expect(client.get()).toEqual(combined); + expect(client.getAll()).toEqual(combined); } }); }); @@ -315,7 +315,7 @@ describe("tests involving null/undefined values", () => { it("creates the dkv twice", async () => { kv1 = await createDkv({ name, noAutosave: true, noCache: true }); kv2 = await createDkv({ name, noAutosave: true, noCache: true }); - expect(kv1.get()).toEqual({}); + expect(kv1.getAll()).toEqual({}); expect(kv1 === kv2).toBe(false); }); @@ -339,7 +339,7 @@ describe("tests involving null/undefined values", () => { expect(kv1.a).toBe(undefined); expect(kv1.a === undefined).toBe(true); expect(kv1.length).toBe(0); - expect(kv1.get()).toEqual({}); + expect(kv1.getAll()).toEqual({}); }); it("make sure undefined (i.e., delete) sync's as expected", async () => { @@ -348,9 +348,6 @@ describe("tests involving null/undefined values", () => { expect(kv2.a).toBe(undefined); expect(kv2.a === undefined).toBe(true); expect(kv2.length).toBe(0); - expect(kv1.get()).toEqual({}); + expect(kv1.getAll()).toEqual({}); }); }); - - - diff --git a/src/packages/backend/nats/test/dstream.test.ts b/src/packages/backend/nats/test/dstream.test.ts index 6822d82a7f..d660674190 100644 --- a/src/packages/backend/nats/test/dstream.test.ts +++ b/src/packages/backend/nats/test/dstream.test.ts @@ -23,14 +23,14 @@ describe("create a dstream and do some basic operations", () => { }); it("starts out empty", () => { - expect(s.get()).toEqual([]); + expect(s.getAll()).toEqual([]); expect(s.length).toEqual(0); }); const mesg = { stdout: "hello" }; it("publishes a message to the stream and confirms it is there", () => { s.push(mesg); - expect(s.get()).toEqual([mesg]); + expect(s.getAll()).toEqual([mesg]); expect(s.length).toEqual(1); expect(s[0]).toEqual(mesg); }); @@ -49,13 +49,13 @@ describe("create a dstream and do some basic operations", () => { // close s: await s.close(); // using s fails - expect(s.get).toThrow("closed"); + expect(s.getAll).toThrow("closed"); // create new stream with same name const t = await createDstream({ name }); // ensure it is NOT just from the cache expect(s === t).toBe(false); // make sure it has our message - expect(t.get()).toEqual([mesg]); + expect(t.getAll()).toEqual([mesg]); }); }); @@ -76,7 +76,7 @@ describe("create two dstreams and observe sync between them", () => { s1.save(); await once(s2, "change"); expect(s2[0]).toEqual("hello"); - expect(s2.get()).toEqual(["hello"]); + expect(s2.getAll()).toEqual(["hello"]); }); it("now write to s2 and save and see that reflected in s1", async () => { @@ -90,8 +90,8 @@ describe("create two dstreams and observe sync between them", () => { s1.push("s1"); s2.push("s2"); // our changes are reflected locally - expect(s1.get()).toEqual(["hello", "hi from s2", "s1"]); - expect(s2.get()).toEqual(["hello", "hi from s2", "s2"]); + expect(s1.getAll()).toEqual(["hello", "hi from s2", "s1"]); + expect(s2.getAll()).toEqual(["hello", "hi from s2", "s2"]); // now kick off the two saves *in parallel* s1.save(); s2.save(); @@ -99,9 +99,9 @@ describe("create two dstreams and observe sync between them", () => { if (s2.length != s1.length) { await once(s2, "change"); } - expect(s1.get()).toEqual(s2.get()); + expect(s1.getAll()).toEqual(s2.getAll()); // in fact s1,s2 is the order since we called s1.save first: - expect(s1.get()).toEqual(["hello", "hi from s2", "s1", "s2"]); + expect(s1.getAll()).toEqual(["hello", "hi from s2", "s1", "s2"]); }); }); @@ -184,7 +184,7 @@ describe("testing start_seq", () => { it("creates a stream and adds 3 messages, noting their assigned sequence numbers", async () => { const s = await createDstream({ name, noAutosave: true }); s.push(1, 2, 3); - expect(s.get()).toEqual([1, 2, 3]); + expect(s.getAll()).toEqual([1, 2, 3]); // save, thus getting sequence numbers s.save(); while (s.seq(2) == null) { @@ -207,13 +207,13 @@ describe("testing start_seq", () => { start_seq: seq[2], }); expect(s.length).toBe(1); - expect(s.get()).toEqual([3]); + expect(s.getAll()).toEqual([3]); }); it("it then pulls in the previous message, so now two messages are loaded", async () => { await s.load({ start_seq: seq[1] }); expect(s.length).toBe(2); - expect(s.get()).toEqual([2, 3]); + expect(s.getAll()).toEqual([2, 3]); }); }); diff --git a/src/packages/backend/nats/test/open-files.test.ts b/src/packages/backend/nats/test/open-files.test.ts index 8235bc944f..9cc1e069dd 100644 --- a/src/packages/backend/nats/test/open-files.test.ts +++ b/src/packages/backend/nats/test/open-files.test.ts @@ -119,7 +119,12 @@ describe("create open file tracker and do some basic operations", () => { expect(o2.get(file2).error.error).toBe("Error: test error"); expect(typeof o2.get(file2).error.time == "number").toBe(true); expect(Math.abs(Date.now() - o2.get(file2).error.time)).toBeLessThan(10000); - o2.save(); + try { + // get a conflict due to above so resolve it... + await o2.save(); + } catch { + o2.save(); + } if (!o1.get(file2).error) { await once(o1, "change", 250); } diff --git a/src/packages/backend/nats/test/service.test.ts b/src/packages/backend/nats/test/service.test.ts index 83e8246e83..4a1f9d45e8 100644 --- a/src/packages/backend/nats/test/service.test.ts +++ b/src/packages/backend/nats/test/service.test.ts @@ -29,7 +29,7 @@ describe("create a service and test it out", () => { } catch (err) { t = `${err}`; } - expect(t).toContain("503"); + expect(t).toContain("Not Available"); }); }); diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 2eb6c47cc9..37d828438b 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -22,11 +22,15 @@ import { isValidUUID } from "@cocalc/util/misc"; import { createOpenFiles, OpenFiles } from "@cocalc/nats/sync/open-files"; import { PubSub } from "@cocalc/nats/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; -import { kv, type KVOptions } from "@cocalc/nats/sync/kv"; -import { dkv, type DKVOptions } from "@cocalc/nats/sync/dkv"; -import { dko, type DKOOptions } from "@cocalc/nats/sync/dko"; -import { stream, type UserStreamOptions } from "@cocalc/nats/sync/stream"; -import { dstream } from "@cocalc/nats/sync/dstream"; +import { kv, type KVOptions, type KV } from "@cocalc/nats/sync/kv"; +import { dkv, type DKVOptions, type DKV } from "@cocalc/nats/sync/dkv"; +import { dko, type DKOOptions, type DKO } from "@cocalc/nats/sync/dko"; +import { + stream, + type UserStreamOptions, + type Stream, +} from "@cocalc/nats/sync/stream"; +import { dstream, type DStream } from "@cocalc/nats/sync/dstream"; import { initApi } from "@cocalc/frontend/nats/api"; import { delay } from "awaiting"; import { Svcm } from "@nats-io/services"; @@ -386,39 +390,49 @@ export class NatsClient { return accumulate; }; - stream = async (opts: Partial & { name: string }) => { + stream = async ( + opts: Partial & { name: string }, + ): Promise> => { if (!opts.account_id && !opts.project_id && opts.limits != null) { throw Error("account client can't set limits on public stream"); } - return await stream({ env: await this.getEnv(), ...opts }); + return await stream({ env: await this.getEnv(), ...opts }); }; - dstream = async (opts: Partial & { name: string }) => { + dstream = async ( + opts: Partial & { name: string }, + ): Promise> => { if (!opts.account_id && !opts.project_id && opts.limits != null) { throw Error("account client can't set limits on public stream"); } return await dstream({ env: await this.getEnv(), ...opts }); }; - kv = async (opts: Partial & { name: string }) => { + kv = async ( + opts: Partial & { name: string }, + ): Promise> => { // if (!opts.account_id && !opts.project_id && opts.limits != null) { // throw Error("account client can't set limits on public stream"); // } - return await kv({ env: await this.getEnv(), ...opts }); + return await kv({ env: await this.getEnv(), ...opts }); }; - dkv = async (opts: Partial & { name: string }) => { + dkv = async ( + opts: Partial & { name: string }, + ): Promise> => { // if (!opts.account_id && !opts.project_id && opts.limits != null) { // throw Error("account client can't set limits on public stream"); // } - return await dkv({ env: await this.getEnv(), ...opts }); + return await dkv({ env: await this.getEnv(), ...opts }); }; - dko = async (opts: Partial & { name: string }) => { + dko = async ( + opts: Partial & { name: string }, + ): Promise> => { // if (!opts.account_id && !opts.project_id && opts.limits != null) { // throw Error("account client can't set limits on public stream"); // } - return await dko({ env: await this.getEnv(), ...opts }); + return await dko({ env: await this.getEnv(), ...opts }); }; microservicesClient = async () => { diff --git a/src/packages/nats/sync/dko.ts b/src/packages/nats/sync/dko.ts index 0037657a94..a8cc610dbf 100644 --- a/src/packages/nats/sync/dko.ts +++ b/src/packages/nats/sync/dko.ts @@ -24,10 +24,10 @@ export interface DKOOptions extends DKVOptions { sep?: string; } -export class DKO extends EventEmitter { +export class DKO extends EventEmitter { opts: DKOOptions; sep: string; - dkv?: DKV; + dkv?: DKV; // can't type this constructor(opts: DKOOptions) { super(); @@ -36,8 +36,11 @@ export class DKO extends EventEmitter { this.init(); return new Proxy(this, { deleteProperty(target, prop) { - target.delete(prop); - return true; + if (typeof prop == "string") { + target.delete(prop); + return true; + } + return false; }, set(target, prop, value) { prop = String(prop); @@ -61,7 +64,7 @@ export class DKO extends EventEmitter { if (this.dkv != null) { throw Error("already initialized"); } - this.dkv = await createDKV({ + this.dkv = await createDKV<{ [key: string]: any }>({ ...this.opts, name: dkoPrefix(this.opts.name), }); @@ -120,7 +123,7 @@ export class DKO extends EventEmitter { return { key, field }; }; - delete = (key) => { + delete = (key: string) => { if (this.dkv == null) { throw Error("closed"); } @@ -138,40 +141,43 @@ export class DKO extends EventEmitter { this.dkv?.clear(); }; - get = (key?) => { + get = (key: string): T | undefined => { if (this.dkv == null) { throw Error("closed"); } - if (key == null) { - // get everything - const all = this.dkv.get(); - const result: any = {}; - for (const x in all) { - const { key, field } = this.fromPath(x); - if (!field) { - continue; - } - if (result[key] == null) { - result[key] = { [field]: all[x] }; - } else { - result[key][field] = all[x]; - } - } - return result; - } else { - const fields = this.dkv.get(key); - if (fields == null) { - return undefined; + const fields = this.dkv.get(key); + if (fields == null) { + return undefined; + } + const x: any = {}; + for (const field of fields) { + x[field] = this.dkv.get(this.toPath(key, field)); + } + return x; + }; + + getAll = (): { [key: string]: T } => { + // get everything + if (this.dkv == null) { + throw Error("closed"); + } + const all = this.dkv.getAll(); + const result: any = {}; + for (const x in all) { + const { key, field } = this.fromPath(x); + if (!field) { + continue; } - const x: any = {}; - for (const field of fields) { - x[field] = this.dkv.get(this.toPath(key, field)); + if (result[key] == null) { + result[key] = { [field]: all[x] }; + } else { + result[key][field] = all[x]; } - return x; } + return result; }; - set = (key: string, obj: any) => { + set = (key: string, obj: T) => { if (this.dkv == null) { throw Error("closed"); } @@ -189,11 +195,11 @@ export class DKO extends EventEmitter { } }; - hasUnsavedChanges = () => { + hasUnsavedChanges = (): boolean => { return !!this.dkv?.hasUnsavedChanges(); }; - unsavedChanges = () => { + unsavedChanges = (): { key: string; field: string }[] => { const dkv = this.dkv; if (dkv == null) { return []; @@ -214,7 +220,7 @@ export class DKO extends EventEmitter { }; } -export const dko = refCache({ +export const cache = refCache({ createKey: userKvKey, createObject: async (opts) => { const k = new DKO(opts); @@ -226,3 +232,7 @@ export const dko = refCache({ function dkoPrefix(name: string): string { return `__dko__${name}`; } + +export async function dko(options: DKOOptions): Promise> { + return await cache(options); +} diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index dbb048ed98..7457a2fa01 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -28,7 +28,7 @@ export interface DKVOptions extends KVOptions { noAutosave?: boolean; } -export class DKV extends EventEmitter { +export class DKV extends EventEmitter { generalDKV?: GeneralDKV; name: string; private prefix: string; @@ -65,7 +65,9 @@ export class DKV extends EventEmitter { this.init(); return new Proxy(this, { deleteProperty(target, prop) { - target.delete(prop); + if (typeof prop == "string") { + target.delete(prop); + } return true; }, set(target, prop, value) { @@ -157,7 +159,7 @@ export class DKV extends EventEmitter { this.removeAllListeners(); }; - delete = (key) => { + delete = (key: string) => { if (this.generalDKV == null) { throw Error("closed"); } @@ -169,7 +171,7 @@ export class DKV extends EventEmitter { }; // server assigned time - time = (key?: string) => { + time = (key?: string): Date | undefined | { [key: string]: Date } => { if (this.generalDKV == null) { throw Error("closed"); } @@ -179,7 +181,7 @@ export class DKV extends EventEmitter { if (key != null || times == null) { return times; } - const obj = this.generalDKV.get(); + const obj = this.generalDKV.getAll(); const x: any = {}; for (const k in obj) { const { key } = obj[k]; @@ -188,36 +190,41 @@ export class DKV extends EventEmitter { return x; }; - has = (key: string) => { + has = (key: string): boolean => { if (this.generalDKV == null) { throw Error("closed"); } return this.generalDKV.has(`${this.prefix}.${this.sha1(key)}`); }; - get = (key?) => { + get = (key: string): T | undefined => { if (this.generalDKV == null) { throw Error("closed"); } - if (key == null) { - const obj = this.generalDKV.get(); - const x: any = {}; - for (const k in obj) { - const { key, value } = obj[k]; - x[key] = value; - } - return x; - } else { - return this.generalDKV.get(`${this.prefix}.${this.sha1(key)}`)?.value; + return this.generalDKV.get(`${this.prefix}.${this.sha1(key)}`)?.value; + }; + + getAll = (): { [key: string]: T } => { + if (this.generalDKV == null) { + throw Error("closed"); } + const obj = this.generalDKV.getAll(); + const x: any = {}; + for (const k in obj) { + const { key, value } = obj[k]; + x[key] = value; + } + return x; }; - get length() { - // not efficient? - return Object.keys(this.get()).length; + get length(): number { + if (this.generalDKV == null) { + throw Error("closed"); + } + return this.generalDKV.length; } - set = (key: string, value: any) => { + set = (key: string, value: T): void => { if (this.generalDKV == null) { throw Error("closed"); } @@ -232,14 +239,14 @@ export class DKV extends EventEmitter { this.generalDKV.set(`${this.prefix}.${this.sha1(key)}`, { key, value }); }; - hasUnsavedChanges = () => { + hasUnsavedChanges = (): boolean => { if (this.generalDKV == null) { return false; } return this.generalDKV.hasUnsavedChanges(); }; - unsavedChanges = () => { + unsavedChanges = (): T[] => { const generalDKV = this.generalDKV; if (generalDKV == null) { return []; @@ -252,7 +259,7 @@ export class DKV extends EventEmitter { }; } -export const dkv = refCache({ +export const cache = refCache({ createKey: userKvKey, createObject: async (opts) => { const k = new DKV(opts); @@ -260,3 +267,7 @@ export const dkv = refCache({ return k; }, }); + +export async function dkv(options: DKVOptions): Promise> { + return await cache(options); +} diff --git a/src/packages/nats/sync/general-dkv.ts b/src/packages/nats/sync/general-dkv.ts index 2cf8d9eb99..b5612def90 100644 --- a/src/packages/nats/sync/general-dkv.ts +++ b/src/packages/nats/sync/general-dkv.ts @@ -17,7 +17,7 @@ is a DKV, you can also access the underlying KV via "store.kv". - The store emits an event ('change', key) whenever anything changes. -- Calling "store.get()" provides ALL the data, and "store.get(key)" gets one value. +- Calling "store.getAll()" provides ALL the data, and "store.get(key)" gets one value. - Use "store.set(key,value)" or "store.set({key:value, key2:value2, ...})" to set data, with the following semantics: @@ -88,8 +88,8 @@ export type MergeFunction = (opts: { remote: any; }) => any; -export class GeneralDKV extends EventEmitter { - private kv?: GeneralKV; +export class GeneralDKV extends EventEmitter { + private kv?: GeneralKV; private jc?; private merge?: MergeFunction; private local: { [key: string]: any } = {}; @@ -204,19 +204,28 @@ export class GeneralDKV extends EventEmitter { this.emit("change", { key, value, prev }); }; - get = (key?) => { + get = (key: string): T | undefined => { if (this.kv == null) { throw Error("closed"); } - if (key != null) { - this.assertValidKey(key); - const local = this.local[key]; - if (local === TOMBSTONE) { - return undefined; - } - return local ?? this.kv.get(key); + this.assertValidKey(key); + const local = this.local[key]; + if (local === TOMBSTONE) { + return undefined; } - const x = { ...this.kv.get(), ...this.local }; + return local ?? this.kv.get(key); + }; + + get length(): number { + // not efficient + return Object.keys(this.getAll()).length; + } + + getAll = (): { [key: string]: T } => { + if (this.kv == null) { + throw Error("closed"); + } + const x = { ...this.kv.getAll(), ...this.local }; for (const key in this.local) { if (this.local[key] === TOMBSTONE) { delete x[key]; @@ -225,11 +234,6 @@ export class GeneralDKV extends EventEmitter { return x; }; - get length() { - // not efficient - return Object.keys(this.get()).length; - } - has = (key: string): boolean => { if (this.kv == null) { throw Error("closed"); @@ -244,14 +248,14 @@ export class GeneralDKV extends EventEmitter { return this.kv.has(key); }; - time = (key?: string) => { + time = (key?: string): { [key: string]: Date } | Date | undefined => { if (this.kv == null) { throw Error("closed"); } return this.kv.time(key); }; - private assertValidKey = (key) => { + private assertValidKey = (key): void => { if (this.kv == null) { throw Error("closed"); } @@ -275,7 +279,7 @@ export class GeneralDKV extends EventEmitter { if (this.kv == null) { throw Error("closed"); } - for (const key in this.kv.get()) { + for (const key in this.kv.getAll()) { this._delete(key); } for (const key in this.local) { diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index 6b286e974c..ddb577424b 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -62,33 +62,33 @@ Type ".help" for more information. > env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/general-kv"); s = new a.GeneralKV({name:'test',env,filter:['foo.>']}); await s.init(); > await s.set({"foo.x":10}) // or s.set("foo.x", 10) -> s.get() +> s.getAll() { 'foo.x': 10 } > await s.delete("foo.x") undefined -> s.get() +> s.getAll() {} > await s.set({"foo.x":10, "foo.bar":20}) // Since the filters are disjoint these are totally different: > t = new a.KV({name:'test',env,filter:['bar.>']}); await t.init(); -> await t.get() +> await t.getAll() {} > await t.set({"bar.abc":10}) undefined -> await t.get() +> await t.getAll() { 'bar.abc': Uint8Array(2) [ 49, 48 ] } -> await s.get() +> await s.getAll() { 'foo.x': 10, 'foo.bar': 20, 'bar.abc': 10 } // The union: > u = new a.KV({name:'test',env,filter:['bar.>', 'foo.>']}); await u.init(); -> u.get() +> u.getAll() { 'foo.x': 10, 'foo.bar': 20, 'bar.abc': 10 } > await s.set({'foo.x':999}) undefined -> u.get() +> u.getAll() { 'foo.x': 999, 'foo.bar': 20, 'bar.abc': 10 } */ @@ -137,14 +137,14 @@ export interface KVLimits { max_msg_size: number; } -export class GeneralKV extends EventEmitter { +export class GeneralKV extends EventEmitter { public readonly name: string; private options?; private filter?: string[]; private env: NatsEnv; private kv?; private watch?; - private all?: { [key: string]: any }; + private all?: { [key: string]: T }; private revisions?: { [key: string]: number }; private times?: { [key: string]: Date }; private sizes?: { [key: string]: number }; @@ -249,26 +249,32 @@ export class GeneralKV extends EventEmitter { this.removeAllListeners(); }; - get = (key?) => { + get = (key: string): T => { if (this.all == null) { throw Error("not initialized"); } - if (key == undefined) { - return { ...this.all }; - } else { - return this.all?.[key]; + return this.all[key]; + }; + + getAll = (): { [key: string]: T } => { + if (this.all == null) { + throw Error("not initialized"); } + return { ...this.all }; }; - get length() { - return Object.keys(this.all ?? {}).length; + get length(): number { + if (this.all == null) { + throw Error("not initialized"); + } + return Object.keys(this.all).length; } has = (key: string): boolean => { return this.all?.[key] !== undefined; }; - time = (key?: string) => { + time = (key?: string): { [key: string]: Date } | Date | undefined => { if (key == null) { return this.times; } else { @@ -276,7 +282,7 @@ export class GeneralKV extends EventEmitter { } }; - assertValidKey = (key: string) => { + assertValidKey = (key: string): void => { if (!this.isValidKey(key)) { throw Error( `delete: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, @@ -284,7 +290,7 @@ export class GeneralKV extends EventEmitter { } }; - isValidKey = (key: string) => { + isValidKey = (key: string): boolean => { if (this.filter == null) { return true; } @@ -296,7 +302,7 @@ export class GeneralKV extends EventEmitter { return false; }; - delete = async (key, revision?) => { + delete = async (key: string, revision?: number) => { this.assertValidKey(key); if ( this.all == null || @@ -370,12 +376,11 @@ export class GeneralKV extends EventEmitter { await awaitMap(Object.keys(this.all), MAX_PARALLEL, this.delete); }; - set = async (...args) => { - if (args.length == 2) { - await this.setOne(args[0], args[1]); - return; - } - const obj = args[0]; + set = async (key: string, value: T) => { + await this.setOne(key, value); + }; + + setMany = async (obj: { [key: string]: T }) => { await awaitMap( Object.keys(obj), MAX_PARALLEL, @@ -383,7 +388,7 @@ export class GeneralKV extends EventEmitter { ); }; - private setOne = async (key, value) => { + private setOne = async (key: string, value: T) => { if (!this.isValidKey(key)) { throw Error( `set: key (=${key}) must match the filter: ${JSON.stringify(this.filter)}`, diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index b698cd2e9a..7e3e2f4587 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -30,8 +30,8 @@ export interface KVOptions { noCache?: boolean; } -export class KV extends EventEmitter { - generalKV?: GeneralKV; +export class KV extends EventEmitter { + generalKV?: GeneralKV<{ key: string; value: T }>; name: string; private prefix: string; private sha1; @@ -52,8 +52,12 @@ export class KV extends EventEmitter { this.init(); return new Proxy(this, { deleteProperty(target, prop) { - target.delete(prop); - return true; + if (typeof prop == "string") { + target.delete(prop); + return true; + } else { + return false; + } }, set(target, prop, value) { prop = String(prop); @@ -90,7 +94,7 @@ export class KV extends EventEmitter { this.removeAllListeners(); }; - delete = async (key) => { + delete = async (key: string) => { if (this.generalKV == null) { throw Error("closed"); } @@ -106,7 +110,7 @@ export class KV extends EventEmitter { }; // server assigned time - time = (key?: string) => { + time = (key?: string): { [key: string]: Date } | Date | undefined => { if (this.generalKV == null) { throw Error("closed"); } @@ -115,24 +119,27 @@ export class KV extends EventEmitter { ); }; - get = (key?) => { + get = (key: string): T | undefined => { if (this.generalKV == null) { throw Error("closed"); } - if (key == null) { - const obj = this.generalKV.get(); - const x: any = {}; - for (const k in obj) { - const { key, value } = obj[k]; - x[key] = value; - } - return x; - } else { - return this.generalKV.get(`${this.prefix}.${this.sha1(key)}`)?.value; + return this.generalKV.get(`${this.prefix}.${this.sha1(key)}`)?.value; + }; + + getAll = (): { [key: string]: T } => { + if (this.generalKV == null) { + throw Error("closed"); + } + const obj = this.generalKV.getAll(); + const x: any = {}; + for (const k in obj) { + const { key, value } = obj[k]; + x[key] = value; } + return x; }; - set = async (key: string, value: any) => { + set = async (key: string, value: T) => { if (this.generalKV == null) { throw Error("closed"); } @@ -151,8 +158,7 @@ export function userKvKey(options: KVOptions) { return JSON.stringify(x); } - -export const kv = refCache({ +export const cache = refCache({ createKey: userKvKey, createObject: async (opts) => { const k = new KV(opts); @@ -160,3 +166,7 @@ export const kv = refCache({ return k; }, }); + +export async function kv(options: KVOptions): Promise> { + return await cache(options); +} diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 32a68612ea..79fd51c39a 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -17,7 +17,7 @@ Change to packages/backend, since packages/nats doesn't have a way to connect: > z.time({path:'a.txt'}) 2025-02-09T16:36:58.510Z > z.touch({path:'foo/b.md',id:0}) -> z.get() +> z.getAll() { 'a.txt': { open: true, count: 3 }, 'foo/b.md': { open: true, count: 1 } @@ -25,7 +25,7 @@ Change to packages/backend, since packages/nats doesn't have a way to connect: Frontend Dev in browser: z = await cc.client.nats_client.openFiles('00847397-d6a8-4cb0-96a8-6ef64ac3e6cf') -z.get() +z.getAll() } */ @@ -172,7 +172,7 @@ export class OpenFiles extends EventEmitter { }; getAll = (): Entry[] => { - const x = this.getDkv().get(); + const x = this.getDkv().getAll(); return Object.keys(x).map((path) => { return { ...x[path], path, time: this.time(path) }; }); diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 12e0249899..54928b1ffd 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -25,7 +25,7 @@ With browser client using a project: # Involving limits: > env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo', env, limits:{max_msgs:5,max_age:1000000*1000*15,max_bytes:10000,max_msg_size:1000}}) -> s.get() +> s.getAll() In browser: > s = await cc.client.nats_client.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo',limits:{max_msgs:5,max_age:1000000*1000*15,max_bytes:10000,max_msg_size:1000}}) @@ -50,6 +50,7 @@ import { isNumericString } from "@cocalc/util/misc"; import { map as awaitMap } from "awaiting"; import { sha1 } from "@cocalc/util/misc"; import refCache from "@cocalc/util/refcache"; +import { type JsMsg } from "@nats-io/jetstream"; class PublishRejectError extends Error { code: string; @@ -112,7 +113,7 @@ export interface StreamOptions { start_seq?: number; } -export class Stream extends EventEmitter { +export class Stream extends EventEmitter { public readonly name: string; public readonly jsname: string; private natsStreamOptions?; @@ -127,8 +128,8 @@ export class Stream extends EventEmitter { private stream?; private watch?; // don't do "this.raw=" or "this.messages=" anywhere in this class! - public readonly raw: any[] = []; - public readonly messages: any[] = []; + public readonly raw: JsMsg[] = []; + public readonly messages: T[] = []; constructor({ name, @@ -208,24 +209,31 @@ export class Stream extends EventEmitter { this.watchForNewData(consumer); }); - get = (n?) => { + get = (n?): T | T[] => { if (this.js == null) { throw Error("closed"); } if (n == null) { - return [...this.messages]; + return this.getAll(); } else { return this.messages[n]; } }; + getAll = (): T[] => { + if (this.js == null) { + throw Error("closed"); + } + return [...this.messages]; + }; + // get server assigned global sequence number of n-th message in stream - seq = (n) => { + seq = (n: number): number | undefined => { return this.raw[n]?.seq; }; // get server assigned time of n-th message in stream - time = (n): Date | undefined => { + time = (n: number): Date | undefined => { const r = this.raw[n]; if (r == null) { return; @@ -233,15 +241,15 @@ export class Stream extends EventEmitter { return new Date(millis(r?.info.timestampNanos)); }; - get length() { + get length(): number { return this.messages.length; } - push = async (...args) => { + push = async (...args: T[]) => { await awaitMap(args, MAX_PARALLEL, this.publish); }; - publish = async (mesg: any, subject?: string, options?) => { + publish = async (mesg: T, subject?: string, options?) => { if (this.js == null) { throw Error("closed"); } @@ -389,7 +397,7 @@ export class Stream extends EventEmitter { } }; - private decode = (raw) => { + private decode = (raw: JsMsg) => { try { return this.env.jc.decode(raw.data); } catch { @@ -398,7 +406,7 @@ export class Stream extends EventEmitter { } }; - private handle = (raw, noEmit = false) => { + private handle = (raw: JsMsg, noEmit = false) => { const mesg = this.decode(raw); this.messages.push(mesg); this.raw.push(raw); @@ -575,7 +583,7 @@ export function userStreamOptionsKey(options: UserStreamOptions) { return JSON.stringify(x); } -export const stream = refCache({ +export const cache = refCache({ createKey: userStreamOptionsKey, createObject: async (options) => { const { account_id, project_id, name } = options; @@ -594,3 +602,9 @@ export const stream = refCache({ return stream; }, }); + +export async function stream( + options: UserStreamOptions, +): Promise> { + return await cache(options); +} diff --git a/src/packages/nats/sync/synctable-kv.ts b/src/packages/nats/sync/synctable-kv.ts index 3a10fabb86..78e5788bbd 100644 --- a/src/packages/nats/sync/synctable-kv.ts +++ b/src/packages/nats/sync/synctable-kv.ts @@ -157,7 +157,7 @@ export class SyncTableKV extends EventEmitter { get = (obj_or_key?) => { if (this.dkv == null) throw Error("closed"); if (obj_or_key == null) { - return this.getHook(this.dkv.get()); + return this.getHook(this.dkv.getAll()); } return this.getHook(this.dkv.get(this.getKey(obj_or_key))); }; @@ -165,7 +165,7 @@ export class SyncTableKV extends EventEmitter { get_one = () => { if (this.dkv == null) throw Error("closed"); // TODO: insanely inefficient, especially if !atomic! - for (const key in this.dkv.get()) { + for (const key in this.dkv.getAll()) { return this.get(key); } }; diff --git a/src/packages/nats/sync/synctable-stream.ts b/src/packages/nats/sync/synctable-stream.ts index 6f09e5acd6..48de9a6d7b 100644 --- a/src/packages/nats/sync/synctable-stream.ts +++ b/src/packages/nats/sync/synctable-stream.ts @@ -94,7 +94,7 @@ export class SyncTableStream extends EventEmitter { this.dstream.on("reject", (err) => { console.warn("synctable-stream: REJECTED - ", err); }); - for (const mesg of this.dstream.get()) { + for (const mesg of this.dstream.getAll()) { this.handle(mesg, false); } this.setState("connected"); diff --git a/src/packages/project/nats/sync.ts b/src/packages/project/nats/sync.ts index df4845078f..7499891c02 100644 --- a/src/packages/project/nats/sync.ts +++ b/src/packages/project/nats/sync.ts @@ -16,24 +16,24 @@ import { export type { Stream, DStream, KV, DKV, OpenFiles, OpenFileEntry }; -export async function stream(opts): Promise { - return await createStream({ project_id, env: await getEnv(), ...opts }); +export async function stream(opts): Promise> { + return await createStream({ project_id, env: await getEnv(), ...opts }); } -export async function dstream(opts): Promise> { +export async function dstream(opts): Promise> { return await createDstream({ project_id, env: await getEnv(), ...opts }); } -export async function kv(opts): Promise { - return await createKV({ project_id, env: await getEnv(), ...opts }); +export async function kv(opts): Promise> { + return await createKV({ project_id, env: await getEnv(), ...opts }); } -export async function dkv(opts): Promise { - return await createDKV({ project_id, env: await getEnv(), ...opts }); +export async function dkv(opts): Promise> { + return await createDKV({ project_id, env: await getEnv(), ...opts }); } -export async function dko(opts): Promise { - return await createDKO({ project_id, env: await getEnv(), ...opts }); +export async function dko(opts): Promise> { + return await createDKO({ project_id, env: await getEnv(), ...opts }); } export async function openFiles(): Promise { From cdf0f1cbc6ed9952e8c74f3f4adac528d8e17532 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 17 Feb 2025 01:05:25 +0000 Subject: [PATCH 219/281] nats: work in progress rewriting listings --- src/packages/frontend/nats/client.ts | 23 +++++++ src/packages/nats/names.ts | 20 +++++- src/packages/nats/service/index.ts | 1 + src/packages/nats/service/listings.ts | 82 +++++++++++++++++++++++++ src/packages/nats/service/service.ts | 21 +++---- src/packages/nats/sync/dko.ts | 4 ++ src/packages/nats/sync/dkv.ts | 20 +++--- src/packages/nats/sync/dstream.ts | 4 ++ src/packages/nats/sync/kv.ts | 22 ++++--- src/packages/nats/sync/stream.ts | 4 ++ src/packages/nats/types.ts | 10 +++ src/packages/project/nats/listings.ts | 87 +++++++++++++++++++++++++++ 12 files changed, 264 insertions(+), 34 deletions(-) create mode 100644 src/packages/nats/service/listings.ts create mode 100644 src/packages/project/nats/listings.ts diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 37d828438b..27eed933ed 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -40,6 +40,11 @@ import type { CallNatsServiceFunction, CreateNatsServiceFunction, } from "@cocalc/nats/service"; +import { + createListingsClient, + getListingsKV, + getListingsTimesKV, +} from "@cocalc/nats/service/listings"; export class NatsClient { client: WebappClient; @@ -441,4 +446,22 @@ export class NatsClient { const svcm = new Svcm(nc); return svcm.client(); }; + + listings = (opts: { project_id: string; compute_server_id?: number }) => { + return createListingsClient(opts); + }; + + getListingsKV = async (opts: { + project_id: string; + compute_server_id?: number; + }) => { + return await getListingsKV(opts); + }; + + getListingsTimesKV = async (opts: { + project_id: string; + compute_server_id?: number; + }) => { + return await getListingsKV(opts); + }; } diff --git a/src/packages/nats/names.ts b/src/packages/nats/names.ts index f892bf2a40..6ae9dc75e1 100644 --- a/src/packages/nats/names.ts +++ b/src/packages/nats/names.ts @@ -13,6 +13,7 @@ For Subjects: import { sha1 } from "@cocalc/util/misc"; import generateVouchers from "@cocalc/util/vouchers"; +import type { Location } from "./types"; // nice alphanumeric string that can be used as nats subject, and very // unlikely to randomly collide with another browser tab from this account. @@ -21,7 +22,7 @@ export function randomId() { } // jetstream name -- we use this canonical name for the KV and the stream associated -// to a project or account. We use the same name for both. +// to a location in cocalc. export function jsName({ project_id, account_id, @@ -45,6 +46,23 @@ export function jsName({ return `account-${account_id}`; } +export function localLocationName({ + compute_server_id, + browser_id, + path, +}: Location): string { + const v: string[] = []; + if (compute_server_id) { + v.push(`id=${compute_server_id}`); + } else if (browser_id) { + v.push(`id=${browser_id}`); + } + if (path) { + v.push(`path=${path}`); + } + return v.join(","); +} + /* Custom inbox prefix per "user"! diff --git a/src/packages/nats/service/index.ts b/src/packages/nats/service/index.ts index 6aa028b588..8007ade3ab 100644 --- a/src/packages/nats/service/index.ts +++ b/src/packages/nats/service/index.ts @@ -3,5 +3,6 @@ export type { CallNatsServiceFunction, ServiceCall, CreateNatsServiceFunction, + NatsService, } from "./service"; export { callNatsService, createNatsService } from "./service"; diff --git a/src/packages/nats/service/listings.ts b/src/packages/nats/service/listings.ts new file mode 100644 index 0000000000..e351fe9d9f --- /dev/null +++ b/src/packages/nats/service/listings.ts @@ -0,0 +1,82 @@ +/* +Service for expressing interest in directory listings in a project or compute server. +*/ + +import { createServiceClient, createServiceHandler } from "./typed"; +import type { DirectoryListingEntry } from "@cocalc/util/types"; +import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; +import { nanos } from "@cocalc/nats/util"; + +export const MAX_FILES_PER_DIRECTORY = 300; + +// discard any listing after this long without update +const MAX_AGE_MS = 1000 * 60 * 60 * 24 * 7; + +// save at most this many directories +const MAX_DIRECTORIES = 50; + +interface ListingsApi { + // cause the directory listing key:value store to watch path + interest: (path: string) => Promise; + + // just directly get the listing info now for this path + getListing: (opts: { + path: string; + hidden?: boolean; + }) => Promise; +} + +interface ListingsOptions { + project_id: string; + compute_server_id?: number; +} + +export function createListingsClient({ + project_id, + compute_server_id = 0, +}: ListingsOptions) { + return createServiceClient({ + project_id, + compute_server_id, + service: "listings", + }); +} + +export type ListingsServiceApi = ReturnType; + +export async function createListingsService({ + project_id, + compute_server_id = 0, + impl, +}: ListingsOptions & { impl }) { + const c = compute_server_id ? ` (compute server: ${compute_server_id})` : ""; + return await createServiceHandler({ + project_id, + compute_server_id, + service: "listings", + description: `Directory listing service: ${c}`, + impl, + }); +} + +export async function getListingsKV( + opts: ListingsOptions, +): Promise> { + return await dkv({ + name: "listings", + limits: { + max_msgs: MAX_DIRECTORIES, + max_age: nanos(MAX_AGE_MS), + }, + ...opts, + }); +} + +export async function getListingsTimesKV( + opts: ListingsOptions, +): Promise> { + return await dkv<{ updated?: number; interest: number }>({ + name: "listings-times", + ...opts, + }); +} diff --git a/src/packages/nats/service/service.ts b/src/packages/nats/service/service.ts index 2750a2964f..eab4439bc1 100644 --- a/src/packages/nats/service/service.ts +++ b/src/packages/nats/service/service.ts @@ -16,7 +16,7 @@ import { type ServiceStats, type ServiceIdentity, } from "@nats-io/services"; -import { type NatsEnv } from "@cocalc/nats/types"; +import { type NatsEnv, type Location } from "@cocalc/nats/types"; import { sha1, trunc_middle } from "@cocalc/util/misc"; import { getEnv } from "@cocalc/nats/client"; import { randomId } from "@cocalc/nats/names"; @@ -24,16 +24,9 @@ import { delay } from "awaiting"; const DEFAULT_TIMEOUT = 5000; -export interface ServiceDescription { +export interface ServiceDescription extends Location { service: string; - project_id?: string; - compute_server_id?: number; - - account_id?: string; - browser_id?: string; - - path?: string; description?: string; // if true and multiple servers are setup in same "location", then they ALL get to respond (sender gets first response). @@ -100,23 +93,23 @@ export function serviceSubject({ path, }: ServiceDescription): string { let segments; - path = path ? sha1(path) : "-"; + path = path ? sha1(path) : "_"; if (!project_id && !account_id) { segments = ["public", service]; } else if (account_id) { segments = [ "services", `account-${account_id}`, - browser_id ?? "-", - project_id ?? "-", - path ?? "-", + browser_id ?? "_", + project_id ?? "_", + path ?? "_", service, ]; } else if (project_id) { segments = [ "services", `project-${project_id}`, - compute_server_id ?? "-", + compute_server_id ?? "_", service, path, ]; diff --git a/src/packages/nats/sync/dko.ts b/src/packages/nats/sync/dko.ts index a8cc610dbf..7cde6fc359 100644 --- a/src/packages/nats/sync/dko.ts +++ b/src/packages/nats/sync/dko.ts @@ -19,6 +19,7 @@ import { dkv as createDKV, DKV, DKVOptions } from "./dkv"; import { userKvKey } from "./kv"; import { is_object } from "@cocalc/util/misc"; import refCache from "@cocalc/util/refcache"; +import { getEnv } from "@cocalc/nats/client"; export interface DKOOptions extends DKVOptions { sep?: string; @@ -223,6 +224,9 @@ export class DKO extends EventEmitter { export const cache = refCache({ createKey: userKvKey, createObject: async (opts) => { + if (opts.env == null) { + opts.env = await getEnv(); + } const k = new DKO(opts); await k.init(); return k; diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index 7457a2fa01..60696fc710 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -19,9 +19,10 @@ import { EventEmitter } from "events"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { GeneralDKV, TOMBSTONE, type MergeFunction } from "./general-dkv"; import { userKvKey, type KVOptions } from "./kv"; -import { jsName } from "@cocalc/nats/names"; +import { jsName, localLocationName } from "@cocalc/nats/names"; import { sha1 } from "@cocalc/util/misc"; import refCache from "@cocalc/util/refcache"; +import { getEnv } from "@cocalc/nats/client"; export interface DKVOptions extends KVOptions { merge?: MergeFunction; @@ -35,22 +36,16 @@ export class DKV extends EventEmitter { private sha1; private opts; - constructor({ - name, - account_id, - project_id, - merge, - env, - noAutosave, - limits, - }: DKVOptions) { + constructor(options: DKVOptions) { super(); + const { name, account_id, project_id, merge, env, noAutosave, limits } = + options; if (env == null) { throw Error("env must not be null"); } // name of the jetstream key:value store. const kvname = jsName({ account_id, project_id }); - this.name = name; + this.name = name + localLocationName(options); this.sha1 = env.sha1 ?? sha1; this.prefix = this.sha1(name); this.opts = { @@ -262,6 +257,9 @@ export class DKV extends EventEmitter { export const cache = refCache({ createKey: userKvKey, createObject: async (opts) => { + if (opts.env == null) { + opts.env = await getEnv(); + } const k = new DKV(opts); await k.init(); return k; diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 215346284c..54868997f4 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -33,6 +33,7 @@ import { sha1 } from "@cocalc/util/misc"; import { millis } from "@cocalc/nats/util"; import refCache from "@cocalc/util/refcache"; import { type JsMsg } from "@nats-io/jetstream"; +import { getEnv } from "@cocalc/nats/client"; const MAX_PARALLEL = 250; @@ -242,6 +243,9 @@ export class DStream extends EventEmitter { const cache = refCache({ createKey: userStreamOptionsKey, createObject: async (options) => { + if (options.env == null) { + options.env = await getEnv(); + } const { account_id, project_id, name } = options; const jsname = jsName({ account_id, project_id }); const subjects = streamSubject({ account_id, project_id }); diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index 7e3e2f4587..916d6c9753 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -14,18 +14,17 @@ Type ".help" for more information. */ import { EventEmitter } from "events"; -import { type NatsEnv } from "@cocalc/nats/types"; +import { type NatsEnv, type Location } from "@cocalc/nats/types"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { GeneralKV, type KVLimits } from "./general-kv"; -import { jsName } from "@cocalc/nats/names"; +import { jsName, localLocationName } from "@cocalc/nats/names"; import { sha1 } from "@cocalc/util/misc"; import refCache from "@cocalc/util/refcache"; +import { getEnv } from "@cocalc/nats/client"; -export interface KVOptions { +export interface KVOptions extends Location { name: string; - account_id?: string; - project_id?: string; - env: NatsEnv; + env?: NatsEnv; limits?: Partial; noCache?: boolean; } @@ -36,11 +35,15 @@ export class KV extends EventEmitter { private prefix: string; private sha1; - constructor({ name, account_id, project_id, env, limits }: KVOptions) { + constructor(options: KVOptions) { super(); + const { name, account_id, project_id, env, limits } = options; // name of the jetstream key:value store. const kvname = jsName({ account_id, project_id }); - this.name = name; + this.name = name + localLocationName(options); + if (env == null) { + throw Error("env must be defined"); + } this.sha1 = env.sha1 ?? sha1; this.prefix = this.sha1(name); this.generalKV = new GeneralKV({ @@ -161,6 +164,9 @@ export function userKvKey(options: KVOptions) { export const cache = refCache({ createKey: userKvKey, createObject: async (opts) => { + if (opts.env == null) { + opts.env = await getEnv(); + } const k = new KV(opts); await k.init(); return k; diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index 54928b1ffd..f56acb6e52 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -51,6 +51,7 @@ import { map as awaitMap } from "awaiting"; import { sha1 } from "@cocalc/util/misc"; import refCache from "@cocalc/util/refcache"; import { type JsMsg } from "@nats-io/jetstream"; +import { getEnv } from "@cocalc/nats/client"; class PublishRejectError extends Error { code: string; @@ -586,6 +587,9 @@ export function userStreamOptionsKey(options: UserStreamOptions) { export const cache = refCache({ createKey: userStreamOptionsKey, createObject: async (options) => { + if (options.env == null) { + options.env = await getEnv(); + } const { account_id, project_id, name } = options; const jsname = jsName({ account_id, project_id }); const subjects = streamSubject({ account_id, project_id }); diff --git a/src/packages/nats/types.ts b/src/packages/nats/types.ts index 375848918b..f3b129253d 100644 --- a/src/packages/nats/types.ts +++ b/src/packages/nats/types.ts @@ -8,3 +8,13 @@ export interface NatsEnv { export type State = "disconnected" | "connected" | "closed"; export type NatsEnvFunction = () => Promise; + +export interface Location { + project_id?: string; + compute_server_id?: number; + + account_id?: string; + browser_id?: string; + + path?: string; +} diff --git a/src/packages/project/nats/listings.ts b/src/packages/project/nats/listings.ts new file mode 100644 index 0000000000..894449870c --- /dev/null +++ b/src/packages/project/nats/listings.ts @@ -0,0 +1,87 @@ +/* Directory Listings + +- A service "listings" in each project and compute server that users call to express + interest in a directory. When there is recent interest in a + directory, we watch it for changes. + +- A DKV store keys paths in the filesystem and values the first + few hundred (ordered by recent) files in that directory, all relative + to the home directory. + +*/ + +import getListing from "@cocalc/backend/get-listing"; +import { + createListingsService, + getListingsKV, + getListingsTimesKV, + MAX_FILES_PER_DIRECTORY, +} from "@cocalc/nats/service/listings"; +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { init as initClient } from "@cocalc/project/client"; +import type { DirectoryListingEntry } from "@cocalc/util/types"; +import { delay } from "awaiting"; +import { type DKV } from "./sync"; +import { type NatsService } from "@cocalc/nats/service"; + +let listings: Listings | null; + +const impl = { + // cause the directory listing key:value store to watch path + interest: async (path: string) => { + while (listings == null) { + await delay(3000); + } + listings.interest(path); + }, + + getListing: async ({ path, hidden }) => { + return await getListing(path, hidden); + }, +}; + +let service: NatsService | null; +export async function init() { + initClient(); + + service = await createListingsService({ + project_id, + compute_server_id, + impl, + }); + const L = new Listings(); + await L.init(); + listings = L; +} + +export async function close() { + service?.close(); + listings?.close(); +} + +class Listings { + private listings: DKV; + private times: DKV<{ + // time last files for a given directory were last updated + updated?: number; + // time user last expressed interest in a given directory + interest: number; + }>; + + init = async () => { + this.listings = await getListingsKV({ project_id, compute_server_id }); + this.times = await getListingsTimesKV({ project_id, compute_server_id }); + }; + + close = () => { + this.listings?.close(); + this.times?.close(); + }; + + interest = async (path: string) => { + this.times.set(path, { ...this.times.get(path), interest: Date.now() }); + + console.log("ignoring ", MAX_FILES_PER_DIRECTORY); + this.listings.set(path, await getListing(path, true)); + }; +} From 3a2c32b799b5e68855fb4fb8d6fd970371e2768e Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 17 Feb 2025 03:04:50 +0000 Subject: [PATCH 220/281] nats: implementing new directory listings watcher --- src/packages/backend/get-listing.ts | 13 +- src/packages/backend/path-watcher.ts | 39 +++++- .../nats-terminal-connection.ts | 2 +- src/packages/frontend/nats/client.ts | 2 +- src/packages/nats/service/listings.ts | 49 +++++--- src/packages/project/nats/listings.ts | 116 ++++++++++++++++-- src/packages/sync-fs/lib/index.ts | 2 +- 7 files changed, 189 insertions(+), 34 deletions(-) diff --git a/src/packages/backend/get-listing.ts b/src/packages/backend/get-listing.ts index fd3dcdae52..0854ea82b1 100644 --- a/src/packages/backend/get-listing.ts +++ b/src/packages/backend/get-listing.ts @@ -19,27 +19,30 @@ Browser client code only uses this through the websocket anyways. import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import type { Dirent, Stats } from "node:fs"; -import { lstat, readdir, readlink, stat } from "node:fs/promises"; +import { lstat, opendir, readdir, readlink, stat } from "node:fs/promises"; import { getLogger } from "./logger"; import { DirectoryListingEntry } from "@cocalc/util/types"; import { join } from "path"; -const logger = getLogger("directory-listing"); +const logger = getLogger("backend:directory-listing"); // SMC_LOCAL_HUB_HOME is used for developing cocalc inside cocalc... const HOME = process.env.SMC_LOCAL_HUB_HOME ?? process.env.HOME ?? ""; const getListing = reuseInFlight( - async( + async ( path: string, // assumed in home directory! hidden: boolean = false, - home = HOME, + { home = HOME, limit }: { home?: string; limit?: number } = {}, ): Promise => { const dir = join(home, path); logger.debug(dir); const files: DirectoryListingEntry[] = []; let file: Dirent; - for (file of await readdir(dir, { withFileTypes: true })) { + for await (file of await opendir(dir)) { + if (limit && files.length >= limit) { + break; + } if (!hidden && file.name[0] === ".") { continue; } diff --git a/src/packages/backend/path-watcher.ts b/src/packages/backend/path-watcher.ts index d7ec85413e..badf5265a4 100644 --- a/src/packages/backend/path-watcher.ts +++ b/src/packages/backend/path-watcher.ts @@ -4,7 +4,7 @@ */ /* -Watch A DIRECTORY for changes of the files in *that* directory only (not recursive). +Watch A DIRECTORY for changes of the files in *that* directory only (not recursive). Use ./watcher.ts for a single file. Slightly generalized fs.watch that works even when the directory doesn't exist, @@ -165,3 +165,40 @@ export class Watcher extends EventEmitter { close(this); } } + +export class MultipathWatcher extends EventEmitter { + private paths: { [path: string]: Watcher } = {}; + private options; + + constructor(options?) { + super(); + this.options = options; + } + + has = (path: string) => { + return this.paths[path] != null; + }; + + add = (path: string) => { + if (this.has(path)) { + // already watching + return; + } + this.paths[path] = new Watcher(path, this.options); + this.paths[path].on("change", () => this.emit("change", path)); + }; + + delete = (path: string) => { + if (!this.has(path)) { + return; + } + this.paths[path].close(); + delete this.paths[path]; + }; + + close = () => { + for (const path in this.paths) { + this.delete(path); + } + }; +} diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index 5926230db9..a19933d518 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -146,7 +146,7 @@ export class NatsTerminalConnection extends EventEmitter { private start = reuseInFlight(async () => { this.setState("init"); try { - await this.api.nats.waitFor({ maxWait: 5000 }); + await this.api.nats.waitFor({ maxWait: 15000 }); await this.api.create(this.options); } catch (err) { this.setState("disconnected"); diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 27eed933ed..a57780e1dc 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -462,6 +462,6 @@ export class NatsClient { project_id: string; compute_server_id?: number; }) => { - return await getListingsKV(opts); + return await getListingsTimesKV(opts); }; } diff --git a/src/packages/nats/service/listings.ts b/src/packages/nats/service/listings.ts index e351fe9d9f..2dfef3446e 100644 --- a/src/packages/nats/service/listings.ts +++ b/src/packages/nats/service/listings.ts @@ -5,15 +5,19 @@ Service for expressing interest in directory listings in a project or compute se import { createServiceClient, createServiceHandler } from "./typed"; import type { DirectoryListingEntry } from "@cocalc/util/types"; import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; -import { nanos } from "@cocalc/nats/util"; -export const MAX_FILES_PER_DIRECTORY = 300; +// record info about at most this many files in a given directory +export const MAX_FILES_PER_DIRECTORY = 10; +//export const MAX_FILES_PER_DIRECTORY = 300; -// discard any listing after this long without update -const MAX_AGE_MS = 1000 * 60 * 60 * 24 * 7; +// cache listing info about at most this many directories +export const MAX_DIRECTORIES = 3; +// export const MAX_DIRECTORIES = 50; -// save at most this many directories -const MAX_DIRECTORIES = 50; +// watch directorie with interest that is this recent +export const INTEREST_CUTOFF_MS = 1000 * 30; + +//export const INTEREST_CUTOFF_MS = 1000 * 60 * 10; interface ListingsApi { // cause the directory listing key:value store to watch path @@ -59,24 +63,41 @@ export async function createListingsService({ }); } +const limits = { + max_msgs: MAX_DIRECTORIES, +}; + +export interface Listing { + files?: DirectoryListingEntry[]; + exists?: boolean; + error?: string; + time: number; + more?: boolean; +} + export async function getListingsKV( opts: ListingsOptions, -): Promise> { - return await dkv({ +): Promise> { + return await dkv({ name: "listings", - limits: { - max_msgs: MAX_DIRECTORIES, - max_age: nanos(MAX_AGE_MS), - }, + limits, ...opts, }); } +export interface Times { + // time last files for a given directory were attempted to be updated + updated?: number; + // time user last expressed interest in a given directory + interest?: number; +} + export async function getListingsTimesKV( opts: ListingsOptions, -): Promise> { - return await dkv<{ updated?: number; interest: number }>({ +): Promise> { + return await dkv({ name: "listings-times", + limits, ...opts, }); } diff --git a/src/packages/project/nats/listings.ts b/src/packages/project/nats/listings.ts index 894449870c..5060abe5a5 100644 --- a/src/packages/project/nats/listings.ts +++ b/src/packages/project/nats/listings.ts @@ -16,13 +16,19 @@ import { getListingsKV, getListingsTimesKV, MAX_FILES_PER_DIRECTORY, + INTEREST_CUTOFF_MS, + type Listing, + type Times, } from "@cocalc/nats/service/listings"; import { compute_server_id, project_id } from "@cocalc/project/data"; import { init as initClient } from "@cocalc/project/client"; -import type { DirectoryListingEntry } from "@cocalc/util/types"; import { delay } from "awaiting"; import { type DKV } from "./sync"; import { type NatsService } from "@cocalc/nats/service"; +import { MultipathWatcher } from "@cocalc/backend/path-watcher"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("project:nats:listings"); let listings: Listings | null; @@ -42,6 +48,7 @@ const impl = { let service: NatsService | null; export async function init() { + logger.debug("init: initializing"); initClient(); service = await createListingsService({ @@ -52,6 +59,7 @@ export async function init() { const L = new Listings(); await L.init(); listings = L; + logger.debug("init: fully ready"); } export async function close() { @@ -60,28 +68,114 @@ export async function close() { } class Listings { - private listings: DKV; - private times: DKV<{ - // time last files for a given directory were last updated - updated?: number; - // time user last expressed interest in a given directory - interest: number; - }>; + private listings: DKV; + + private times: DKV; + + private watcher: MultipathWatcher; + + private state: "init" | "ready" | "closed" = "init"; + + constructor() { + this.watcher = new MultipathWatcher(); + this.watcher.on("change", this.updateListing); + } init = async () => { + logger.debug("Listings.init: start"); this.listings = await getListingsKV({ project_id, compute_server_id }); this.times = await getListingsTimesKV({ project_id, compute_server_id }); + // start watching paths with recent interest + const cutoff = Date.now() - INTEREST_CUTOFF_MS; + const times = this.times.getAll(); + for (const path in times) { + if ((times[path].interest ?? 0) >= cutoff) { + await this.updateListing(path); + } + } + this.monitorInterestLoop(); + this.state = "ready"; + logger.debug("Listings.init: done"); + }; + + private monitorInterestLoop = async () => { + while (this.state != "closed") { + const cutoff = Date.now() - INTEREST_CUTOFF_MS; + const times = this.times.getAll(); + for (const path in times) { + if ((times[path].interest ?? 0) <= cutoff) { + if (this.watcher.has(path)) { + logger.debug("monitorInterestLoop: stop watching", { path }); + this.watcher.delete(path); + } + } + } + await delay(30 * 1000); + } }; close = () => { + this.state = "closed"; + this.watcher.close(); this.listings?.close(); this.times?.close(); }; + updateListing = async (path: string) => { + logger.debug("updateListing", { path }); + path = canonicalPath(path); + this.watcher.add(canonicalPath(path)); + const start = Date.now(); + try { + let files = await getListing(path, true, { + limit: MAX_FILES_PER_DIRECTORY + 1, + }); + const more = files.length == MAX_FILES_PER_DIRECTORY + 1; + if (more) { + files = files.slice(0, MAX_FILES_PER_DIRECTORY); + } + this.listings.set(path, { + files, + exists: true, + time: Date.now(), + more, + }); + logger.debug("updateListing: success", { + path, + ms: Date.now() - start, + count: files.length, + more, + }); + } catch (err) { + let error = `${err}`; + if (error.startsWith("Error: ")) { + error = error.slice("Error: ".length); + } + this.listings.set(path, { + error, + time: Date.now(), + exists: error.includes("ENOENT") ? false : undefined, + }); + logger.debug("updateListing: error", { + path, + ms: Date.now() - start, + error, + }); + } + }; + interest = async (path: string) => { + logger.debug("interest", { path }); + path = canonicalPath(path); this.times.set(path, { ...this.times.get(path), interest: Date.now() }); - - console.log("ignoring ", MAX_FILES_PER_DIRECTORY); - this.listings.set(path, await getListing(path, true)); + this.updateListing(path); }; } + +// this does a tiny amount to make paths more canonical. +function canonicalPath(path: string): string { + if (path == "." || path == "~") { + return ""; + } + return path; +} diff --git a/src/packages/sync-fs/lib/index.ts b/src/packages/sync-fs/lib/index.ts index 996b551595..70bdcd7d7c 100644 --- a/src/packages/sync-fs/lib/index.ts +++ b/src/packages/sync-fs/lib/index.ts @@ -315,7 +315,7 @@ class SyncFS { } case "listing": - return await getListing(data.path, data.hidden, this.mount); + return await getListing(data.path, data.hidden, { HOME: this.mount }); case "exec": if (data.opts.command == "cc-new-file") { From 2e9c5439b98da7bcbc1eab28c44cc4c8d0a223f8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 17 Feb 2025 12:20:32 +0000 Subject: [PATCH 221/281] nats listings: packaged client --- src/packages/backend/path-watcher.ts | 2 +- src/packages/frontend/nats/client.ts | 23 ++-------- src/packages/nats/service/listings.ts | 61 +++++++++++++++++++++++++-- src/packages/nats/service/service.ts | 13 +++++- src/packages/nats/sync/dkv.ts | 2 +- src/packages/nats/sync/kv.ts | 2 +- src/packages/project/nats/listings.ts | 23 ++++++++-- 7 files changed, 94 insertions(+), 32 deletions(-) diff --git a/src/packages/backend/path-watcher.ts b/src/packages/backend/path-watcher.ts index badf5265a4..71e03c6958 100644 --- a/src/packages/backend/path-watcher.ts +++ b/src/packages/backend/path-watcher.ts @@ -54,7 +54,7 @@ const logger = getLogger("backend:path-watcher"); const POLLING = true; const DEFAULT_POLL_MS = parseInt( - process.env.COCALC_FS_WATCHER_POLL_INTERVAL_MS ?? "3000", + process.env.COCALC_FS_WATCHER_POLL_INTERVAL_MS ?? "2000", ); const ChokidarOpts: WatchOptions = { diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index a57780e1dc..4f24a0d680 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -40,11 +40,7 @@ import type { CallNatsServiceFunction, CreateNatsServiceFunction, } from "@cocalc/nats/service"; -import { - createListingsClient, - getListingsKV, - getListingsTimesKV, -} from "@cocalc/nats/service/listings"; +import { listingsClient } from "@cocalc/nats/service/listings"; export class NatsClient { client: WebappClient; @@ -440,28 +436,17 @@ export class NatsClient { return await dko({ env: await this.getEnv(), ...opts }); }; - microservicesClient = async () => { + microservices = async () => { const nc = await this.getConnection(); // @ts-ignore const svcm = new Svcm(nc); return svcm.client(); }; - listings = (opts: { project_id: string; compute_server_id?: number }) => { - return createListingsClient(opts); - }; - - getListingsKV = async (opts: { - project_id: string; - compute_server_id?: number; - }) => { - return await getListingsKV(opts); - }; - - getListingsTimesKV = async (opts: { + listings = async (opts: { project_id: string; compute_server_id?: number; }) => { - return await getListingsTimesKV(opts); + return await listingsClient(opts); }; } diff --git a/src/packages/nats/service/listings.ts b/src/packages/nats/service/listings.ts index 2dfef3446e..be9d4cb199 100644 --- a/src/packages/nats/service/listings.ts +++ b/src/packages/nats/service/listings.ts @@ -1,10 +1,11 @@ /* -Service for expressing interest in directory listings in a project or compute server. +Service for watching directory listings in a project or compute server. */ import { createServiceClient, createServiceHandler } from "./typed"; import type { DirectoryListingEntry } from "@cocalc/util/types"; import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; +import { EventEmitter } from "events"; // record info about at most this many files in a given directory export const MAX_FILES_PER_DIRECTORY = 10; @@ -19,9 +20,9 @@ export const INTEREST_CUTOFF_MS = 1000 * 30; //export const INTEREST_CUTOFF_MS = 1000 * 60 * 10; -interface ListingsApi { +export interface ListingsApi { // cause the directory listing key:value store to watch path - interest: (path: string) => Promise; + watch: (path: string) => Promise; // just directly get the listing info now for this path getListing: (opts: { @@ -88,7 +89,7 @@ export async function getListingsKV( export interface Times { // time last files for a given directory were attempted to be updated updated?: number; - // time user last expressed interest in a given directory + // time user requested to watch a given directory interest?: number; } @@ -101,3 +102,55 @@ export async function getListingsTimesKV( ...opts, }); } + +/* Unified interface to the above components for clients */ + +export class ListingsClient extends EventEmitter { + options: { project_id: string; compute_server_id: number }; + api: ListingsApi; + times: DKV; + listings: DKV; + + constructor({ + project_id, + compute_server_id = 0, + }: { + project_id: string; + compute_server_id?: number; + }) { + super(); + this.options = { project_id, compute_server_id }; + } + + init = async () => { + this.api = createListingsClient(this.options); + this.times = await getListingsTimesKV(this.options); + this.listings = await getListingsKV(this.options); + this.listings.on("change", (path) => this.emit("change", path)); + }; + + get = (path: string): Listing | undefined => { + return this.listings.get(path); + }; + + getAll = () => this.listings.getAll(); + + close = () => { + this.times.close(); + this.listings.close(); + }; + + watch = async (path) => { + await this.api.watch(path); + }; + + getListing = async (opts) => { + return await this.api.getListing(opts); + }; +} + +export async function listingsClient(options) { + const C = new ListingsClient(options); + await C.init(); + return C; +} diff --git a/src/packages/nats/service/service.ts b/src/packages/nats/service/service.ts index eab4439bc1..7d5d2d65d7 100644 --- a/src/packages/nats/service/service.ts +++ b/src/packages/nats/service/service.ts @@ -262,7 +262,16 @@ export async function waitForNatsService({ let d = 100; let m = 100; const start = Date.now(); - let ping = await pingNatsService({ options, maxWait: m }); + const getPing = async (m: number) => { + try { + return await pingNatsService({ options, maxWait: m }); + } catch { + // ping can fail, e.g, if not connected to nats at all or the ping + // service isn't up yet. + return [] as ServiceIdentity[]; + } + }; + let ping = await getPing(m); while (ping.length == 0) { d = Math.min(10000, d * 1.3); m = Math.min(1500, m * 1.3); @@ -271,7 +280,7 @@ export async function waitForNatsService({ throw Error("timeout"); } await delay(d); - ping = await pingNatsService({ options, maxWait: m }); + ping = await getPing(m); } return ping; } diff --git a/src/packages/nats/sync/dkv.ts b/src/packages/nats/sync/dkv.ts index 60696fc710..934f5f4cf6 100644 --- a/src/packages/nats/sync/dkv.ts +++ b/src/packages/nats/sync/dkv.ts @@ -47,7 +47,7 @@ export class DKV extends EventEmitter { const kvname = jsName({ account_id, project_id }); this.name = name + localLocationName(options); this.sha1 = env.sha1 ?? sha1; - this.prefix = this.sha1(name); + this.prefix = this.sha1(this.name); this.opts = { name: kvname, filter: `${this.prefix}.>`, diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index 916d6c9753..83f891ab5e 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -45,7 +45,7 @@ export class KV extends EventEmitter { throw Error("env must be defined"); } this.sha1 = env.sha1 ?? sha1; - this.prefix = this.sha1(name); + this.prefix = this.sha1(this.name); this.generalKV = new GeneralKV({ name: kvname, filter: `${this.prefix}.>`, diff --git a/src/packages/project/nats/listings.ts b/src/packages/project/nats/listings.ts index 5060abe5a5..57748f2cc3 100644 --- a/src/packages/project/nats/listings.ts +++ b/src/packages/project/nats/listings.ts @@ -8,6 +8,21 @@ few hundred (ordered by recent) files in that directory, all relative to the home directory. + +DEVELOPMENT: + +1. Setup project environment variables as usual + +2. Stop listings service running in the project: + + define COCALC_PROJECT_ID, COCALC_NATS_JWT, etc., as usual. + +3. Start your own server + +.../src/packages/project/nats$ node + +> await require('@cocalc/project/nats/listings').init() + */ import getListing from "@cocalc/backend/get-listing"; @@ -34,11 +49,11 @@ let listings: Listings | null; const impl = { // cause the directory listing key:value store to watch path - interest: async (path: string) => { + watch: async (path: string) => { while (listings == null) { await delay(3000); } - listings.interest(path); + listings.watch(path); }, getListing: async ({ path, hidden }) => { @@ -164,8 +179,8 @@ class Listings { } }; - interest = async (path: string) => { - logger.debug("interest", { path }); + watch = async (path: string) => { + logger.debug("watch", { path }); path = canonicalPath(path); this.times.set(path, { ...this.times.get(path), interest: Date.now() }); this.updateListing(path); From 0338401e5f045ed691e62b5f41d782b64181003d Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 17 Feb 2025 14:02:47 +0000 Subject: [PATCH 222/281] nats listings -- start integrating with frontend --- .../frontend/project/directory-listing.ts | 2 +- .../explorer/file-listing/file-listing.tsx | 2 +- .../frontend/project/nats/listings.ts | 171 ++++++++++++++++++ .../frontend/project/page/flyouts/files.tsx | 2 +- src/packages/frontend/project_store.ts | 6 +- src/packages/nats/service/listings.ts | 56 ++++-- src/packages/project/client.ts | 2 +- src/packages/project/nats/api/index.ts | 5 + src/packages/project/nats/index.ts | 2 + src/packages/project/nats/listings.ts | 42 +++-- 10 files changed, 247 insertions(+), 43 deletions(-) create mode 100644 src/packages/frontend/project/nats/listings.ts diff --git a/src/packages/frontend/project/directory-listing.ts b/src/packages/frontend/project/directory-listing.ts index 4473b70e7d..90eff2d8d2 100644 --- a/src/packages/frontend/project/directory-listing.ts +++ b/src/packages/frontend/project/directory-listing.ts @@ -114,7 +114,7 @@ export async function get_directory_listing(opts: ListingOpts) { } } -import { Listings } from "./websocket/listings"; +import { Listings } from "./nats/listings"; export async function get_directory_listing2(opts: ListingOpts): Promise { log("get_directory_listing2", opts); diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index e1bbab9b13..79e60df2b7 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -18,7 +18,7 @@ import { useTypedRedux, } from "@cocalc/frontend/app-framework"; import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; -import { WATCH_THROTTLE_MS } from "@cocalc/frontend/project/websocket/listings"; +import { WATCH_THROTTLE_MS } from "@cocalc/frontend/project/nats/listings"; import { ProjectActions } from "@cocalc/frontend/project_actions"; import { MainConfiguration } from "@cocalc/frontend/project_configuration"; import { FileRow } from "./file-row"; diff --git a/src/packages/frontend/project/nats/listings.ts b/src/packages/frontend/project/nats/listings.ts new file mode 100644 index 0000000000..ee06af6fdf --- /dev/null +++ b/src/packages/frontend/project/nats/listings.ts @@ -0,0 +1,171 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +import { TypedMap } from "@cocalc/frontend/app-framework"; +import { webapp_client } from "@cocalc/frontend/webapp-client"; +import { Listing } from "@cocalc/util/db-schema/listings"; +import type { DirectoryListingEntry } from "@cocalc/util/types"; +import { EventEmitter } from "events"; +import { fromJS, List } from "immutable"; +import { + listingsClient, + type ListingsClient, + createListingsApiClient, + type ListingsApi, + MIN_INTEREST_INTERVAL_MS, +} from "@cocalc/nats/service/listings"; + +export const WATCH_THROTTLE_MS = MIN_INTEREST_INTERVAL_MS; + +type ImmutablePathEntry = TypedMap; + +type State = "init" | "ready" | "closed"; + +export type ImmutableListing = TypedMap; + +export class Listings extends EventEmitter { + private project_id: string; + private compute_server_id: number; + private state: State = "init"; + private listingsClient?: ListingsClient; + private api: ListingsApi; + + constructor(project_id: string, compute_server_id: number = 0) { + super(); + this.project_id = project_id; + this.compute_server_id = compute_server_id; + this.api = createListingsApiClient({ project_id, compute_server_id }); + this.init(); + } + + private init = async () => { + this.listingsClient = await listingsClient({ + project_id: this.project_id, + compute_server_id: this.compute_server_id, + }); + this.listingsClient.on("change", (path) => { + this.emit("change", [path]); + }); + // [ ] TODO: delete event for deleted paths + this.setState("ready"); + }; + + // Watch directory for changes. + watch = async (path: string, force?): Promise => { + this.listingsClient?.watch(path, force); + }; + + get = async ( + path: string, + trigger_start_project?: boolean, + ): Promise => { + const x = this.listingsClient?.get(path); + if (x != null) { + if (x.error) { + throw Error(x.error); + } + if (!x.exists) { + throw Error(`ENOENT: no such directory '${path}'`); + } + return x.files; + } + // TODO: trigger_start_project ? + return await this.api.getListing({ path, hidden: true }); + }; + + getDeleted = async (path: string): Promise | undefined> => { + // todo + return fromJS([]); + }; + + undelete = async (path: string): Promise => {}; + + // true or false if known deleted or not; undefined if don't know yet. + // TODO: technically we should check the all the + // deleted_file_variations... but that is really an edge case + // that probably doesn't matter much. + public isDeleted = (filename: string): boolean | undefined => {}; + + // Does a call to the project to directly determine whether or + // not the given path exists. This doesn't depend on the table. + // Can throw an exception if it can't contact the project. + exists = async (path: string): Promise => { + return ( + ( + await webapp_client.exec({ + project_id: this.project_id, + command: "test", + args: ["-e", path], + err_on_exit: false, + }) + ).exit_code == 0 + ); + }; + + // Returns: + // - List in case of a proper directory listing + // - string in case of an error + // - undefined if directory listing not known (and error not known either). + getForStore = async ( + path: string, + ): Promise | undefined | string> => { + try { + const x = await this.get(path); + return fromJS(x) as unknown as List; + } catch (err) { + return `${err}`; + } + }; + + getUsingDatabase = async ( + path: string, + ): Promise => { + return this.listingsClient?.get(path)?.files; + }; + + // TODO: we now only know there are more, not how many + getMissingUsingDatabase = async ( + path: string, + ): Promise => { + return this.listingsClient?.get(path)?.more ? 1 : 0; + }; + + getMissing = (path: string): number | undefined => { + return this.listingsClient?.get(path)?.more ? 1 : 0; + }; + + getListingDirectly = async ( + path: string, + trigger_start_project?: boolean, + ): Promise => { + // todo: trigger_start_project + return await this.api.getListing({ path, hidden: true }); + }; + + close = (): void => { + if (this.state == "closed") { + return; + } + this.setState("closed"); + this.listingsClient?.close(); + delete this.listingsClient; + }; + + isReady = (): boolean => { + return this.state == ("ready" as State); + }; + + setState = (state: State) => { + this.state = state; + this.emit(state); + }; +} + +export function listings( + project_id: string, + compute_server_id: number = 0, +): Listings { + return new Listings(project_id, compute_server_id); +} diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index 6809905350..250a57b67e 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -35,7 +35,7 @@ import { DirectoryListingEntry, FileMap, } from "@cocalc/frontend/project/explorer/types"; -import { WATCH_THROTTLE_MS } from "@cocalc/frontend/project/websocket/listings"; +import { WATCH_THROTTLE_MS } from "@cocalc/frontend/project/nats/listings"; import { mutate_data_to_compute_public_files } from "@cocalc/frontend/project_store"; import track from "@cocalc/frontend/user-tracking"; import { diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index b4f223ccd4..02f2916e04 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -27,10 +27,7 @@ import { TypedMap, } from "@cocalc/frontend/app-framework"; import { ProjectLogMap } from "@cocalc/frontend/project/history/types"; -import { - Listings, - listings, -} from "@cocalc/frontend/project/websocket/listings"; +import { Listings, listings } from "@cocalc/frontend/project/nats/listings"; import { FILE_ACTIONS, ProjectActions, @@ -649,6 +646,7 @@ export class ProjectStore extends Store { } }); listingsTable.on("change", async (paths) => { + console.log("change", paths); let directory_listings_for_server = this.getIn(["directory_listings", computeServerId]) ?? immutable.Map(); diff --git a/src/packages/nats/service/listings.ts b/src/packages/nats/service/listings.ts index be9d4cb199..5ea652329b 100644 --- a/src/packages/nats/service/listings.ts +++ b/src/packages/nats/service/listings.ts @@ -8,17 +8,18 @@ import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; import { EventEmitter } from "events"; // record info about at most this many files in a given directory -export const MAX_FILES_PER_DIRECTORY = 10; -//export const MAX_FILES_PER_DIRECTORY = 300; +//export const MAX_FILES_PER_DIRECTORY = 10; +export const MAX_FILES_PER_DIRECTORY = 500; // cache listing info about at most this many directories -export const MAX_DIRECTORIES = 3; -// export const MAX_DIRECTORIES = 50; +//export const MAX_DIRECTORIES = 3; +export const MAX_DIRECTORIES = 50; -// watch directorie with interest that is this recent -export const INTEREST_CUTOFF_MS = 1000 * 30; +// watch directories with interest that are this recent +//export const INTEREST_CUTOFF_MS = 1000 * 30; +export const INTEREST_CUTOFF_MS = 1000 * 60 * 10; -//export const INTEREST_CUTOFF_MS = 1000 * 60 * 10; +export const MIN_INTEREST_INTERVAL_MS = 15 * 1000; export interface ListingsApi { // cause the directory listing key:value store to watch path @@ -36,7 +37,7 @@ interface ListingsOptions { compute_server_id?: number; } -export function createListingsClient({ +export function createListingsApiClient({ project_id, compute_server_id = 0, }: ListingsOptions) { @@ -47,7 +48,7 @@ export function createListingsClient({ }); } -export type ListingsServiceApi = ReturnType; +export type ListingsServiceApi = ReturnType; export async function createListingsService({ project_id, @@ -108,8 +109,8 @@ export async function getListingsTimesKV( export class ListingsClient extends EventEmitter { options: { project_id: string; compute_server_id: number }; api: ListingsApi; - times: DKV; - listings: DKV; + times?: DKV; + listings?: DKV; constructor({ project_id, @@ -123,24 +124,45 @@ export class ListingsClient extends EventEmitter { } init = async () => { - this.api = createListingsClient(this.options); + this.api = createListingsApiClient(this.options); this.times = await getListingsTimesKV(this.options); this.listings = await getListingsKV(this.options); - this.listings.on("change", (path) => this.emit("change", path)); + this.listings.on("change", ({ key: path }) => this.emit("change", path)); }; get = (path: string): Listing | undefined => { + if (this.listings == null) { + throw Error("not ready"); + } return this.listings.get(path); }; - getAll = () => this.listings.getAll(); + getAll = () => { + if (this.listings == null) { + throw Error("not ready"); + } + + this.listings.getAll(); + }; close = () => { - this.times.close(); - this.listings.close(); + this.times?.close(); + delete this.times; + this.listings?.close(); + delete this.listings; }; - watch = async (path) => { + watch = async (path, force = false) => { + if (this.times == null) { + throw Error("not ready"); + } + if (!force) { + const last = this.times.get(path)?.interest ?? 0; + if (Math.abs(Date.now() - last) < MIN_INTEREST_INTERVAL_MS) { + // somebody already expressed interest very recently + return; + } + } await this.api.watch(path); }; diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index f5a415979a..31a5012c1f 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -90,7 +90,7 @@ let client: Client; export function init() { if (client != null) { - throw Error("BUG: Client already initialized!"); + return client; } client = new Client(); setNatsClient(client); diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index 0afaa563fe..b3ff410911 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -42,6 +42,7 @@ import { type ProjectApi } from "@cocalc/nats/project-api"; import getConnection from "@cocalc/project/nats/connection"; import { getSubject } from "../names"; import { terminate as terminateOpenFiles } from "@cocalc/project/nats/open-files"; +import { close as closeListings } from "@cocalc/project/nats/listings"; import { Svcm } from "@nats-io/services"; import { compute_server_id, project_id } from "@cocalc/project/data"; @@ -77,6 +78,10 @@ async function listen(api, subject) { terminateOpenFiles(); mesg.respond(jc.encode({ status: "terminated", service })); continue; + } else if (service == "listings") { + closeListings(); + mesg.respond(jc.encode({ status: "terminated", service })); + continue; } else if (service == "api") { // special hook so admin can terminate handling. This is useful for development. console.warn("TERMINATING listening on ", subject); diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts index d194215bdb..7f2bdaae84 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/nats/index.ts @@ -11,6 +11,7 @@ import { init as initAPI } from "./api"; import { init as initOpenFiles } from "./open-files"; // TODO: initWebsocketApi is temporary import { init as initWebsocketApi } from "./browser-websocket-api"; +import { init as initListings } from "./listings"; const logger = getLogger("project:nats:index"); @@ -19,4 +20,5 @@ export default async function init() { await initAPI(); await initOpenFiles(); initWebsocketApi(); + await initListings(); } diff --git a/src/packages/project/nats/listings.ts b/src/packages/project/nats/listings.ts index 57748f2cc3..de3e4b25b2 100644 --- a/src/packages/project/nats/listings.ts +++ b/src/packages/project/nats/listings.ts @@ -11,9 +11,15 @@ DEVELOPMENT: -1. Setup project environment variables as usual +1. Stop listings service running in the project by running this in your browser: + + await cc.client.nats_client.projectApi({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'}).system.terminate({service:'listings'}) + + {status: 'terminated', service: 'listings'} + + +2. Setup project environment variables as usual. -2. Stop listings service running in the project: define COCALC_PROJECT_ID, COCALC_NATS_JWT, etc., as usual. @@ -45,22 +51,6 @@ import getLogger from "@cocalc/backend/logger"; const logger = getLogger("project:nats:listings"); -let listings: Listings | null; - -const impl = { - // cause the directory listing key:value store to watch path - watch: async (path: string) => { - while (listings == null) { - await delay(3000); - } - listings.watch(path); - }, - - getListing: async ({ path, hidden }) => { - return await getListing(path, hidden); - }, -}; - let service: NatsService | null; export async function init() { logger.debug("init: initializing"); @@ -82,6 +72,22 @@ export async function close() { listings?.close(); } +let listings: Listings | null; + +const impl = { + // cause the directory listing key:value store to watch path + watch: async (path: string) => { + while (listings == null) { + await delay(3000); + } + listings.watch(path); + }, + + getListing: async ({ path, hidden }) => { + return await getListing(path, hidden); + }, +}; + class Listings { private listings: DKV; From f1dbea2fb53456c024c5682c03793d63cbb16696 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 17 Feb 2025 14:37:52 +0000 Subject: [PATCH 223/281] make 'cc.current()' better to aid in debugability of cocalc --- src/packages/frontend/app-framework/index.ts | 26 ++++++++++++------- src/packages/frontend/client/console.ts | 1 + .../terminal-editor/connected-terminal.ts | 2 +- src/packages/project/nats/api/index.ts | 2 +- src/packages/project/nats/listings.ts | 2 +- src/packages/project/nats/open-files.ts | 4 +-- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/packages/frontend/app-framework/index.ts b/src/packages/frontend/app-framework/index.ts index 579e65db73..300c8dc640 100644 --- a/src/packages/frontend/app-framework/index.ts +++ b/src/packages/frontend/app-framework/index.ts @@ -219,25 +219,33 @@ export class AppRedux extends AppReduxBase { // getEditorActions but for whatever editor -- this is mainly meant to be used // from the console when debugging, e.g., smc.redux.currentEditorActions() - public currentEditor(): { + public currentEditor = (): { actions: Actions | undefined; store: Store | undefined; - } { + } => { const project_id = this.getStore("page").get("active_top_tab"); + const current: { + project_id?: string; + path?: string; + account_id?: string; + actions?; + store?; + } = { account_id: this.getStore("account")?.get("account_id") }; if (!is_valid_uuid_string(project_id)) { - return { actions: undefined, store: undefined }; + return current; } + current.project_id = project_id; const store = this.getProjectStore(project_id); const tab = store.get("active_project_tab"); if (!tab.startsWith("editor-")) { - return { actions: undefined, store: undefined }; + return current; } const path = tab.slice("editor-".length); - return { - actions: this.getEditorActions(project_id, path), - store: this.getEditorStore(project_id, path), - }; - } + current.path = path; + current.actions = this.getEditorActions(project_id, path); + current.store = this.getEditorStore(project_id, path); + return current; + }; } const computed = (rtype) => { diff --git a/src/packages/frontend/client/console.ts b/src/packages/frontend/client/console.ts index 4043fe2f47..4e54923230 100644 --- a/src/packages/frontend/client/console.ts +++ b/src/packages/frontend/client/console.ts @@ -50,6 +50,7 @@ export function setup_global_cocalc(client): void { cocalc.done = cocalc.misc.done; cocalc.schema = require("@cocalc/util/schema"); cocalc.redux = redux; + cocalc.current = redux.currentEditor; cocalc.load_eruda = load_eruda; cocalc.compute = require("@cocalc/frontend/compute/api"); console.log( diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index b08467332e..e63bb5a9ce 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -726,7 +726,7 @@ export class Terminal { // during initial load return; } - const geom = this.fitAddon.proposeDimensions(); + const geom = this.fitAddon?.proposeDimensions(); if (geom == null) { return; } diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index b3ff410911..c320c7f3c0 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -3,7 +3,7 @@ How to do development (so in a dev project doing cc-in-cc dev). 0. From the browser, terminate this api server running in the project: - > await cc.client.nats_client.projectApi({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'}).system.terminate({service:'api'}) + > await cc.client.nats_client.projectApi(cc.current()).system.terminate({service:'api'}) {status: 'terminated', service: 'api'} diff --git a/src/packages/project/nats/listings.ts b/src/packages/project/nats/listings.ts index de3e4b25b2..8214e695dc 100644 --- a/src/packages/project/nats/listings.ts +++ b/src/packages/project/nats/listings.ts @@ -13,7 +13,7 @@ DEVELOPMENT: 1. Stop listings service running in the project by running this in your browser: - await cc.client.nats_client.projectApi({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'}).system.terminate({service:'listings'}) + await cc.client.nats_client.projectApi(cc.current()).system.terminate({service:'listings'}) {status: 'terminated', service: 'listings'} diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index e0d8bd6ea7..b4e6801b13 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -3,9 +3,9 @@ Handle opening files in a project to save/load from disk and also enable compute DEVELOPMENT: -0. From the browser, terminate open-files api service running in the project already, if any +0. From the browser with the project opened, terminate the open-files api service: - await cc.client.nats_client.projectApi({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'}).system.terminate({service:'open-files'}) + await cc.client.nats_client.projectApi(cc.current()).system.terminate({service:'open-files'}) // {status: 'terminated', service: 'open-files'} From b16a35c523bf38515af239090e46aadcf2a8eb02 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 17 Feb 2025 15:27:01 +0000 Subject: [PATCH 224/281] nats: work in progress on revamping directory listings state --- src/packages/frontend/app-framework/index.ts | 11 +++-- .../project/fetch-directory-listing.ts | 10 ++++- .../frontend/project/nats/listings.ts | 43 +++++++++++++++---- src/packages/frontend/project_store.ts | 1 - src/packages/nats/service/listings.ts | 19 +++++++- 5 files changed, 67 insertions(+), 17 deletions(-) diff --git a/src/packages/frontend/app-framework/index.ts b/src/packages/frontend/app-framework/index.ts index 300c8dc640..fb1d04e29a 100644 --- a/src/packages/frontend/app-framework/index.ts +++ b/src/packages/frontend/app-framework/index.ts @@ -220,16 +220,19 @@ export class AppRedux extends AppReduxBase { // getEditorActions but for whatever editor -- this is mainly meant to be used // from the console when debugging, e.g., smc.redux.currentEditorActions() public currentEditor = (): { - actions: Actions | undefined; - store: Store | undefined; + project_id?: string; + path?: string; + account_id?: string; + actions?: Actions; + store?: Store; } => { const project_id = this.getStore("page").get("active_top_tab"); const current: { project_id?: string; path?: string; account_id?: string; - actions?; - store?; + actions?: Actions; + store?: Store; } = { account_id: this.getStore("account")?.get("account_id") }; if (!is_valid_uuid_string(project_id)) { return current; diff --git a/src/packages/frontend/project/fetch-directory-listing.ts b/src/packages/frontend/project/fetch-directory-listing.ts index e2e5c90000..b1300437c4 100644 --- a/src/packages/frontend/project/fetch-directory-listing.ts +++ b/src/packages/frontend/project/fetch-directory-listing.ts @@ -5,8 +5,8 @@ import { get_directory_listing } from "./directory-listing"; import { fromJS, Map } from "immutable"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -//const log = (...args) => console.log("fetchDirectoryListing", ...args); -const log = (..._args) => {}; +const log = (...args) => console.log("fetchDirectoryListing", ...args); +//const log = (..._args) => {}; interface FetchDirectoryListingOpts { path?: string; @@ -41,6 +41,12 @@ const fetchDirectoryListing = reuseInFlight( if (store == null) { return; } + if (!is_running_or_starting(actions.project_id)) { + // can't do anything if project isn't running + store.get_listings(); // call this to at least ensure listings info is loaded. + return; + } + const { force } = opts; const path = getPath(actions, opts); const compute_server_id = getComputeServerId(actions, opts); diff --git a/src/packages/frontend/project/nats/listings.ts b/src/packages/frontend/project/nats/listings.ts index ee06af6fdf..1c43ba0d3b 100644 --- a/src/packages/frontend/project/nats/listings.ts +++ b/src/packages/frontend/project/nats/listings.ts @@ -48,19 +48,26 @@ export class Listings extends EventEmitter { this.listingsClient.on("change", (path) => { this.emit("change", [path]); }); + // cause load of all cached data into redux + this.emit("change", Object.keys(this.listingsClient.getAll())); // [ ] TODO: delete event for deleted paths this.setState("ready"); }; // Watch directory for changes. watch = async (path: string, force?): Promise => { - this.listingsClient?.watch(path, force); + try { + await this.listingsClient?.watch(path, force); + } catch {} }; get = async ( path: string, trigger_start_project?: boolean, ): Promise => { + if (this.listingsClient == null) { + throw Error("listings not ready"); + } const x = this.listingsClient?.get(path); if (x != null) { if (x.error) { @@ -76,17 +83,27 @@ export class Listings extends EventEmitter { }; getDeleted = async (path: string): Promise | undefined> => { - // todo - return fromJS([]); + if (this.listingsClient == null) { + throw Error("listings not ready"); + } + const x = this.listingsClient.get(path); + return fromJS(x?.deleted); }; - undelete = async (path: string): Promise => {}; + undelete = async (filename: string): Promise => { + if (this.listingsClient == null) { + throw Error("listings not ready"); + } + this.listingsClient.undelete(filename); + }; // true or false if known deleted or not; undefined if don't know yet. // TODO: technically we should check the all the // deleted_file_variations... but that is really an edge case // that probably doesn't matter much. - public isDeleted = (filename: string): boolean | undefined => {}; + public isDeleted = (filename: string): boolean | undefined => { + return this.listingsClient?.isDeleted(filename); + }; // Does a call to the project to directly determine whether or // not the given path exists. This doesn't depend on the table. @@ -122,24 +139,34 @@ export class Listings extends EventEmitter { getUsingDatabase = async ( path: string, ): Promise => { - return this.listingsClient?.get(path)?.files; + if (this.listingsClient == null) { + throw Error("listings not ready"); + } + return this.listingsClient.get(path)?.files; }; // TODO: we now only know there are more, not how many getMissingUsingDatabase = async ( path: string, ): Promise => { - return this.listingsClient?.get(path)?.more ? 1 : 0; + if (this.listingsClient == null) { + throw Error("listings not ready"); + } + return this.listingsClient.get(path)?.more ? 1 : 0; }; getMissing = (path: string): number | undefined => { - return this.listingsClient?.get(path)?.more ? 1 : 0; + if (this.listingsClient == null) { + throw Error("listings not ready"); + } + return this.listingsClient.get(path)?.more ? 1 : 0; }; getListingDirectly = async ( path: string, trigger_start_project?: boolean, ): Promise => { + console.trace("getListingDirectly", { path }); // todo: trigger_start_project return await this.api.getListing({ path, hidden: true }); }; diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 02f2916e04..16e9b102dc 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -646,7 +646,6 @@ export class ProjectStore extends Store { } }); listingsTable.on("change", async (paths) => { - console.log("change", paths); let directory_listings_for_server = this.getIn(["directory_listings", computeServerId]) ?? immutable.Map(); diff --git a/src/packages/nats/service/listings.ts b/src/packages/nats/service/listings.ts index 5ea652329b..207f96558e 100644 --- a/src/packages/nats/service/listings.ts +++ b/src/packages/nats/service/listings.ts @@ -75,6 +75,7 @@ export interface Listing { error?: string; time: number; more?: boolean; + deleted?: string[]; } export async function getListingsKV( @@ -141,8 +142,7 @@ export class ListingsClient extends EventEmitter { if (this.listings == null) { throw Error("not ready"); } - - this.listings.getAll(); + return this.listings.getAll(); }; close = () => { @@ -169,6 +169,21 @@ export class ListingsClient extends EventEmitter { getListing = async (opts) => { return await this.api.getListing(opts); }; + + getDeleted = () => {}; + + isDeleted = (filename: string) => { + return false; + }; + + undelete = (path: string) => { + let deleted = this.getDeleted(path) ?? []; + if (!deleted.includes(path)) { + return; + } + deleted = deleted.filter((x) => x != path); + this.listings.set(path); + }; } export async function listingsClient(options) { From 708fe11b54c3c87a376937d7d26e70d17d9c129c Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 17 Feb 2025 23:56:48 +0000 Subject: [PATCH 225/281] open-files: dealing with deleted files - MUCH better than before! Finally.... sanity. --- src/packages/backend/misc/async-utils-node.ts | 15 +--- src/packages/frontend/client/client.ts | 10 ++- src/packages/frontend/nats/client.ts | 43 ++++++++++- .../frontend/project/nats/listings.ts | 9 ++- src/packages/nats/service/listings.ts | 15 ---- src/packages/nats/sync/dstream.ts | 2 +- src/packages/nats/sync/open-files.ts | 48 ++++++++++-- src/packages/nats/sync/stream.ts | 8 +- src/packages/project/nats/open-files.ts | 75 ++++++++++++++++++- 9 files changed, 175 insertions(+), 50 deletions(-) diff --git a/src/packages/backend/misc/async-utils-node.ts b/src/packages/backend/misc/async-utils-node.ts index 641f42095a..b908568cb3 100644 --- a/src/packages/backend/misc/async-utils-node.ts +++ b/src/packages/backend/misc/async-utils-node.ts @@ -3,16 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ -import { access, readFile, unlink } from "node:fs/promises"; +import { readFile, unlink } from "node:fs/promises"; +import { pathExists } from "fs-extra"; -export async function exists(path: string): Promise { - // fs.exists is deprecated - try { - await access(path); - return true; - } catch { - return false; - } -} - -export { readFile, unlink }; +export { readFile, unlink, pathExists as exists }; diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index daf9807d8b..d0ad03ceb8 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -366,14 +366,20 @@ class Client extends EventEmitter implements WebappClient { touchOpenFile = async ({ project_id, path, - // id, + setNotDeleted, + // id }: { project_id: string; path: string; id?: number; + // if file is deleted, this explicitly undeletes it. + setNotDeleted?: boolean; }) => { const x = await this.nats_client.openFiles(project_id); - await x.touch(path); + if (setNotDeleted) { + x.setNotDeleted(path); + } + x.touch(path); }; } diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 4f24a0d680..d1affba8bd 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -1,5 +1,6 @@ import * as nats from "nats.ws"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; +import { redux } from "@cocalc/frontend/app-framework"; import type { WebappClient } from "@cocalc/frontend/client/client"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { join } from "path"; @@ -338,13 +339,29 @@ export class NatsClient { openFiles = reuseInFlight(async (project_id: string) => { if (this.openFilesCache[project_id] == null) { - this.openFilesCache[project_id] = await createOpenFiles({ + const openFiles = await createOpenFiles({ project_id, env: await this.getEnv(), }); - this.openFilesCache[project_id].on("closed", () => { + this.openFilesCache[project_id] = openFiles; + openFiles.on("closed", () => { delete this.openFilesCache[project_id]; }); + openFiles.on("change", (entry) => { + if (entry.deleted) { + // if this file is opened here, close it. + ensureClosed({ project_id, path: entry.path, openFiles }); + } + }); + for (const entry of openFiles.getAll()) { + if (entry.deleted) { + ensureClosed({ + project_id, + path: entry.path, + openFiles, + }); + } + } } return this.openFilesCache[project_id]!; }); @@ -450,3 +467,25 @@ export class NatsClient { return await listingsClient(opts); }; } + +async function ensureClosed({ project_id, path, openFiles }) { + // console.log("ensureClosed", { path }); + if (!redux.hasProjectStore(project_id)) { + // console.log("ensureClosed: project not opened"); + // file can't be opened if project isn't opened + return; + } + const store = redux.getProjectStore(project_id); + if (!store.get("open_files").has(path)) { + // console.log("ensureClosed: file not opened"); + return; + } + if (await store.get_listings().exists(path)) { + // console.log("ensureClosed: file exists on disk -- setting not deleted"); + openFiles.setNotDeleted(path); + return; + } + // console.log("ensureClosed: closing"); + const actions = redux.getProjectActions(project_id); + actions.close_tab(path); +} diff --git a/src/packages/frontend/project/nats/listings.ts b/src/packages/frontend/project/nats/listings.ts index 1c43ba0d3b..292bc84089 100644 --- a/src/packages/frontend/project/nats/listings.ts +++ b/src/packages/frontend/project/nats/listings.ts @@ -86,15 +86,15 @@ export class Listings extends EventEmitter { if (this.listingsClient == null) { throw Error("listings not ready"); } - const x = this.listingsClient.get(path); - return fromJS(x?.deleted); + // TODO -- or not + return undefined; }; undelete = async (filename: string): Promise => { if (this.listingsClient == null) { throw Error("listings not ready"); } - this.listingsClient.undelete(filename); + // TODO }; // true or false if known deleted or not; undefined if don't know yet. @@ -102,7 +102,8 @@ export class Listings extends EventEmitter { // deleted_file_variations... but that is really an edge case // that probably doesn't matter much. public isDeleted = (filename: string): boolean | undefined => { - return this.listingsClient?.isDeleted(filename); + // TODO + return false; }; // Does a call to the project to directly determine whether or diff --git a/src/packages/nats/service/listings.ts b/src/packages/nats/service/listings.ts index 207f96558e..1fc277df58 100644 --- a/src/packages/nats/service/listings.ts +++ b/src/packages/nats/service/listings.ts @@ -169,21 +169,6 @@ export class ListingsClient extends EventEmitter { getListing = async (opts) => { return await this.api.getListing(opts); }; - - getDeleted = () => {}; - - isDeleted = (filename: string) => { - return false; - }; - - undelete = (path: string) => { - let deleted = this.getDeleted(path) ?? []; - if (!deleted.includes(path)) { - return; - } - deleted = deleted.filter((x) => x != path); - this.listings.set(path); - }; } export async function listingsClient(options) { diff --git a/src/packages/nats/sync/dstream.ts b/src/packages/nats/sync/dstream.ts index 54868997f4..fdadb5d601 100644 --- a/src/packages/nats/sync/dstream.ts +++ b/src/packages/nats/sync/dstream.ts @@ -12,7 +12,7 @@ Type ".help" for more information. > s = await require("@cocalc/backend/nats/sync").dstream({name:'test'}); -> s = await require("@cocalc/backend/nats/sync").dstream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo'});0 +> s = await require("@cocalc/backend/nats/sync").dstream({project_id:cc.current().project_id,name:'foo'});0 */ diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 79fd51c39a..98d9dfb8f5 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -7,7 +7,7 @@ Change to packages/backend, since packages/nats doesn't have a way to connect: ~/cocalc/src/packages/backend$ node -> z = await require('@cocalc/backend/nats/sync').openFiles('00847397-d6a8-4cb0-96a8-6ef64ac3e6cf') +> z = await require('@cocalc/backend/nats/sync').openFiles({project_id:cc.current().project_id) > z.touch({path:'a.txt'}) > z.get({path:'a.txt'}) { open: true, count: 1, time:2025-02-09T16:37:20.713Z } @@ -24,7 +24,7 @@ Change to packages/backend, since packages/nats doesn't have a way to connect: Frontend Dev in browser: -z = await cc.client.nats_client.openFiles('00847397-d6a8-4cb0-96a8-6ef64ac3e6cf') +z = await cc.client.nats_client.openFiles({project_id:cc.current().project_id)) z.getAll() } */ @@ -34,8 +34,8 @@ import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; import { nanos } from "@cocalc/nats/util"; import { EventEmitter } from "events"; -// 1 week -const MAX_AGE_MS = 1000 * 60 * 60 * 168; +// 1 day +const MAX_AGE_MS = 1000 * 60 * 60 * 24; export interface Entry { // path to file relative to HOME @@ -48,6 +48,13 @@ export interface Entry { // https://github.com/nats-io/nats-server/discussions/3095 time?: Date; count?: number; + // if the file was removed from disk (and not immmediately written back), + // then deleted gets set to the time when this happened (in ms since epoch) + // and the file is closed on the backend. It won't be re-opened until + // either (1) the file is created on disk again, or (2) deleted is cleared. + // Note: the actual time here isn't really important -- what matter is the number + // is nonzero. It's just used for a display to the user. + deleted?: number; } interface Options { @@ -91,7 +98,7 @@ export class OpenFiles extends EventEmitter { }; init = async () => { - const d = await dkv({ + const d = await dkv({ name: "open-files", project_id: this.project_id, env: this.env, @@ -100,6 +107,11 @@ export class OpenFiles extends EventEmitter { }, noAutosave: this.noAutosave, noCache: this.noCache, + merge: ({ local, remote }) => { + // resolve conflicts by merging object state. This is important so, e.g., the + // deleted state doesn't get overwritten on reconnect by clients that didn't know. + return { ...remote, ...local }; + }, }); this.dkv = d; d.on("change", ({ key: path }) => { @@ -144,8 +156,12 @@ export class OpenFiles extends EventEmitter { const dkv = this.getDkv(); // n = sequence number to make sure a write happens, which updates // server assigned timestamp. - const count = dkv.get(path)?.count ?? 0; - dkv.set(path, { open: true, count: count + 1 }); + const cur = dkv.get(path); + dkv.set(path, { + ...cur, + open: true, + count: (cur?.count ?? 0) + 1, + }); }; setError = (path: string, err?: any) => { @@ -171,6 +187,24 @@ export class OpenFiles extends EventEmitter { dkv.set(path, { ...dkv.get(path), open: false }); }; + setDeleted = (path: string) => { + const dkv = this.getDkv(); + dkv.set(path, { ...dkv.get(path), deleted: Date.now() }); + }; + + isDeleted = (path: string) => { + return !!this.getDkv().get(path)?.deleted; + }; + + setNotDeleted = (path: string) => { + const dkv = this.getDkv(); + const cur = dkv.get(path); + if (cur == null) { + return; + } + dkv.set(path, { ...cur, deleted: undefined }); + }; + getAll = (): Entry[] => { const x = this.getDkv().getAll(); return Object.keys(x).map((path) => { diff --git a/src/packages/nats/sync/stream.ts b/src/packages/nats/sync/stream.ts index f56acb6e52..b69263094f 100644 --- a/src/packages/nats/sync/stream.ts +++ b/src/packages/nats/sync/stream.ts @@ -16,19 +16,19 @@ Type ".help" for more information. With browser client using a project: # in browser -> s = await cc.client.nats_client.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo'}) +> s = await cc.client.nats_client.stream({project_id:cc.current().project_id,name:'foo'}) # in node: -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo', env}) +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:cc.current().project_id,name:'foo', env}) # Involving limits: -> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo', env, limits:{max_msgs:5,max_age:1000000*1000*15,max_bytes:10000,max_msg_size:1000}}) +> env = await require("@cocalc/backend/nats/env").getEnv(); a = require("@cocalc/nats/sync/stream"); s = await a.stream({project_id:cc.current().project_id,name:'foo', env, limits:{max_msgs:5,max_age:1000000*1000*15,max_bytes:10000,max_msg_size:1000}}) > s.getAll() In browser: -> s = await cc.client.nats_client.stream({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094',name:'foo',limits:{max_msgs:5,max_age:1000000*1000*15,max_bytes:10000,max_msg_size:1000}}) +> s = await cc.client.nats_client.stream({project_id:cc.current().project_id, name:'foo',limits:{max_msgs:5,max_age:1000000*1000*15,max_bytes:10000,max_msg_size:1000}}) TODO: - maybe the limits and other config should be stored in a KV store so diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index b4e6801b13..0c39f4b820 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -13,7 +13,7 @@ DEVELOPMENT: Set env variables as in a project (see api/index.ts ), then in nodejs: -DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:nats:open-files node +DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:nats:* node > x = await require("@cocalc/project/nats/open-files").init(); Object.keys(x) [ 'openFiles', 'openDocs', 'formatter', 'terminate' ] @@ -49,12 +49,16 @@ import { get_blob_store } from "@cocalc/jupyter/blobs"; import { createFormatterService } from "./formatter"; import { type NatsService } from "@cocalc/nats/service/service"; import { createTerminalService } from "./terminal"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { map as awaitMap } from "awaiting"; // ensure nats connection stuff is initialized import "@cocalc/backend/nats"; const logger = getLogger("project:nats:open-files"); +const FILE_DELETION_CHECK_INTERVAL_MS = 2000; + let openFiles: OpenFiles | null = null; let formatter: any = null; const openDocs: { [path: string]: SyncDoc | NatsService } = {}; @@ -72,6 +76,10 @@ export async function init() { // start loop to watch for and close files that aren't touched frequently: closeIgnoredFilesLoop(); + // watch if any file that is currently opened on this host gets deleted, + // and if so, mark it as such, and set it to closed. + watchForFileDeletionLoop(); + // handle changes openFiles.on("change", (entry) => { handleChange(entry); @@ -99,10 +107,21 @@ function getCutoff() { return new Date(Date.now() - 2.5 * NATS_OPEN_FILE_TOUCH_INTERVAL); } -async function handleChange({ path, open, time }: OpenFileEntry) { - logger.debug("handleChange", { path, open, time }); +async function handleChange({ path, open, time, deleted }: OpenFileEntry) { + logger.debug("handleChange", { path, open, time, deleted }); const syncDoc = openDocs[path]; const isOpenHere = syncDoc != null; + if (deleted) { + if (await exists(path)) { + // it's back + openFiles?.setNotDeleted(path); + } else { + if (isOpenHere) { + closeDoc(path); + } + return; + } + } // TODO: need another table with compute server mappings // const id = 0; // todo // if (id != compute_server_id) { @@ -171,6 +190,56 @@ async function closeIgnoredFilesLoop() { } } +async function checkForFileDeletion(path: string) { + if (openFiles == null) { + return; + } + const entry = openFiles.get(path); + if (entry == null) { + return; + } + if (entry.deleted) { + // already set as deleted -- shouldn't still be opened + await closeDoc(entry.path); + } else { + // if file doesn't exist and still doesn't exist in 1 second, + // mark deleted, which also causes a close. + if (await exists(entry.path)) { + return; + } + // doesn't exist + await delay(250); + if (await exists(entry.path)) { + return; + } + // still doesn't exist + if (openFiles != null) { + openFiles.setDeleted(entry.path); + await closeDoc(entry.path); + } + } +} + +async function watchForFileDeletionLoop() { + while (openFiles != null && openFiles.state == "connected") { + await delay(FILE_DELETION_CHECK_INTERVAL_MS); + if (openFiles?.state != "connected") { + return; + } + const paths = Object.keys(openDocs); + if (paths.length == 0) { + logger.debug("watchForFileDeletionLoop: no paths currently open"); + continue; + } + logger.debug( + "watchForFileDeletionLoop: checking", + paths.length, + "currently open paths to see if any were deleted", + ); + await awaitMap(paths, 20, checkForFileDeletion); + } +} + const closeDoc = reuseInFlight(async (path: string) => { logger.debug("close", { path }); const doc = openDocs[path]; From 1b760e4a08625535f7063ec6db5e15defb760cdb Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 18 Feb 2025 02:34:59 +0000 Subject: [PATCH 226/281] nats: add monitoring to kv store - this should really be built into nats... but it isn't and without it, things break horribly! --- src/packages/nats/sync/general-kv.ts | 53 ++++++++++++++++++++++++- src/packages/project/nats/open-files.ts | 10 +++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index ddb577424b..8099e029bb 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -105,6 +105,7 @@ import { isEqual } from "lodash"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { map as awaitMap } from "awaiting"; import { throttle } from "lodash"; +import { delay } from "awaiting"; class RejectError extends Error { code: string; @@ -113,6 +114,8 @@ class RejectError extends Error { const MAX_PARALLEL = 250; +const WATCH_MONITOR_INTERVAL = 15 * 1000; + // Note that the limit options are named in exactly the same was as for streams, // which is convenient for consistency. This is not consistent with NATS's // own KV store limit naming. @@ -149,6 +152,7 @@ export class GeneralKV extends EventEmitter { private times?: { [key: string]: Date }; private sizes?: { [key: string]: number }; private limits: KVLimits; + private revision: number = 0; constructor({ name, @@ -194,6 +198,7 @@ export class GeneralKV extends EventEmitter { key: this.filter, }); this.revisions = revisions; + this.revision = Math.max(0, ...Object.values(await revisions)); this.times = times; this.sizes = {}; for (const key in all) { @@ -203,17 +208,22 @@ export class GeneralKV extends EventEmitter { this.all = all; this.emit("connected"); this.startWatch(); + this.monitorWatch(); }); - private startWatch = async () => { + private startWatch = async ({ + resumeFromRevision, + }: { resumeFromRevision?: number } = {}) => { // watch for changes this.watch = await this.kv.watch({ ignoreDeletes: false, include: "updates", key: this.filter, + resumeFromRevision, }); for await (const x of this.watch) { const { revision, key, value, sm } = x; + this.revision = revision; if ( this.revisions == null || this.all == null || @@ -239,8 +249,49 @@ export class GeneralKV extends EventEmitter { } }; + // For both the kv and streams as best I can tell we MUST periodically poll the + // server to see if the watch is still working. If not we create a new one + // starting at the last revision we got. + private monitorWatch = async () => { + while (this.revisions != null) { + await delay(WATCH_MONITOR_INTERVAL); + if (this.revisions == null) { + return; + } + if (this.watch == null) { + continue; + } + try { + await this.watch._data.info(); + } catch (err) { + if (this.revisions == null) { + return; + } + if ( + err.name == "ConsumerNotFoundError" || + err.code == 10014 || + err.message == "consumer not found" + ) { + // if it is a consumer not found error, we make a new watch, + // starting AFTER the last revision we retrieved + this.watch.stop(); // stop current watch + // make new watch: + const resumeFromRevision = this.revision + ? this.revision + 1 + : undefined; + this.startWatch({ resumeFromRevision }); + } + } + } + }; + close = () => { + if (this.revisions == null) { + // already closed + return; + } this.watch?.stop(); + delete this.watch; delete this.all; delete this.times; delete this.revisions; diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 0c39f4b820..281190e864 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -51,6 +51,7 @@ import { type NatsService } from "@cocalc/nats/service/service"; import { createTerminalService } from "./terminal"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { map as awaitMap } from "awaiting"; +import { unlink } from "fs/promises"; // ensure nats connection stuff is initialized import "@cocalc/backend/nats"; @@ -216,6 +217,15 @@ async function checkForFileDeletion(path: string) { if (openFiles != null) { openFiles.setDeleted(entry.path); await closeDoc(entry.path); + // closing a file may cause it to try to save to disk the last version, + // so we delete it if that happens. + // TODO: add an option to close everywhere to not do this, and/or make + // it not save on close if the file doesn't exist. + try { + if (await exists(entry.path)) { + await unlink(entry.path); + } + } catch {} } } } From eec42ba8680e84a1e8bdd889816e1bdf84ff6bd4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 18 Feb 2025 03:13:43 +0000 Subject: [PATCH 227/281] nats kv: confirmed that my monitor works --- src/packages/nats/sync/general-kv.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/packages/nats/sync/general-kv.ts b/src/packages/nats/sync/general-kv.ts index 8099e029bb..5021f32b5e 100644 --- a/src/packages/nats/sync/general-kv.ts +++ b/src/packages/nats/sync/general-kv.ts @@ -261,12 +261,22 @@ export class GeneralKV extends EventEmitter { if (this.watch == null) { continue; } + // To see this happen, get the open files, then delete the consumer associated + // to the watch: + // This is in a browser with a project opened: + // + // o = await cc.client.nats_client.openFiles(cc.current().project_id) + // o.dkv.generalDKV.kv.watch._data.delete() + // + // Now observe that "await o.dkv.generalDKV.kv.watch._data.info()" fails as below, + // but within a few seconds everything is fine again. + try { await this.watch._data.info(); } catch (err) { if (this.revisions == null) { - return; - } + return; + } if ( err.name == "ConsumerNotFoundError" || err.code == 10014 || From ca7ea3c9cab97a897a696758375742b4db6f03a7 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 18 Feb 2025 15:17:49 +0000 Subject: [PATCH 228/281] nats-based delete file management: deal with some edge cases exposed by terminals --- .../nats-terminal-connection.ts | 2 ++ src/packages/nats/sync/open-files.ts | 25 +++++++++++++------ src/packages/project/nats/open-files.ts | 23 +++++++++++++---- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts index a19933d518..5ac44a8066 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/nats-terminal-connection.ts @@ -109,6 +109,8 @@ export class NatsTerminalConnection extends EventEmitter { touchLoop = async ({ project_id, path }) => { while (this.state != ("closed" as State)) { try { + // this marks the path as being of interest for editing and starts + // the service; it doesn't actually create a file on disk. await webapp_client.touchOpenFile({ project_id, path, diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 98d9dfb8f5..297bb08ece 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -110,6 +110,7 @@ export class OpenFiles extends EventEmitter { merge: ({ local, remote }) => { // resolve conflicts by merging object state. This is important so, e.g., the // deleted state doesn't get overwritten on reconnect by clients that didn't know. + //console.log("merge", local, remote, { ...remote, ...local }); return { ...remote, ...local }; }, }); @@ -146,6 +147,14 @@ export class OpenFiles extends EventEmitter { return dkv; }; + private set = (key, entry: Entry) => { + // remove time field, if there -- it is filled in automatically + entry = { ...entry }; + // @ts-ignore + delete entry.time; + this.getDkv().set(key, entry); + }; + // When a client has a file open, they should periodically // touch it to indicate that it is open. // updates timestamp and ensures open=true. @@ -157,7 +166,7 @@ export class OpenFiles extends EventEmitter { // n = sequence number to make sure a write happens, which updates // server assigned timestamp. const cur = dkv.get(path); - dkv.set(path, { + this.set(path, { ...cur, open: true, count: (cur?.count ?? 0) + 1, @@ -169,11 +178,11 @@ export class OpenFiles extends EventEmitter { if (!err) { const current = { ...dkv.get(path) }; delete current.error; - dkv.set(path, current); + this.set(path, current); } else { const current = { ...dkv.get(path) }; current.error = { time: Date.now(), error: `${err}` }; - dkv.set(path, current); + this.set(path, current); } }; @@ -184,12 +193,12 @@ export class OpenFiles extends EventEmitter { // can just immediately reopen the file. closeFile = (path: string) => { const dkv = this.getDkv(); - dkv.set(path, { ...dkv.get(path), open: false }); + this.set(path, { ...dkv.get(path), open: false }); }; setDeleted = (path: string) => { const dkv = this.getDkv(); - dkv.set(path, { ...dkv.get(path), deleted: Date.now() }); + this.set(path, { ...dkv.get(path), deleted: Date.now() }); }; isDeleted = (path: string) => { @@ -198,11 +207,13 @@ export class OpenFiles extends EventEmitter { setNotDeleted = (path: string) => { const dkv = this.getDkv(); - const cur = dkv.get(path); + let cur = dkv.get(path); if (cur == null) { return; } - dkv.set(path, { ...cur, deleted: undefined }); + cur = { ...cur }; + delete cur.deleted; + this.set(path, cur); }; getAll = (): Entry[] => { diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 281190e864..3d236fbfb0 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -52,6 +52,7 @@ import { createTerminalService } from "./terminal"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { map as awaitMap } from "awaiting"; import { unlink } from "fs/promises"; +import { join } from "path"; // ensure nats connection stuff is initialized import "@cocalc/backend/nats"; @@ -195,6 +196,12 @@ async function checkForFileDeletion(path: string) { if (openFiles == null) { return; } + if (path.endsWith(".term")) { + // term files are exempt -- we don't save data in them and often + // don't actually make the hidden ones for each frame in the + // filesystem at all. + return; + } const entry = openFiles.get(path); if (entry == null) { return; @@ -203,27 +210,33 @@ async function checkForFileDeletion(path: string) { // already set as deleted -- shouldn't still be opened await closeDoc(entry.path); } else { + if (!process.env.HOME) { + // too dangerous + return; + } + const fullPath = join(process.env.HOME, entry.path); // if file doesn't exist and still doesn't exist in 1 second, // mark deleted, which also causes a close. - if (await exists(entry.path)) { + if (await exists(fullPath)) { return; } // doesn't exist await delay(250); - if (await exists(entry.path)) { + if (await exists(fullPath)) { return; } // still doesn't exist if (openFiles != null) { + logger.debug("checkForFileDeletion: marking as deleted -- ", entry); openFiles.setDeleted(entry.path); - await closeDoc(entry.path); + await closeDoc(fullPath); // closing a file may cause it to try to save to disk the last version, // so we delete it if that happens. // TODO: add an option to close everywhere to not do this, and/or make // it not save on close if the file doesn't exist. try { - if (await exists(entry.path)) { - await unlink(entry.path); + if (await exists(fullPath)) { + await unlink(fullPath); } } catch {} } From 4214fec56bea503feafd811557e0237c4d40d318 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 18 Feb 2025 16:37:21 +0000 Subject: [PATCH 229/281] nats file deletion handling: work in progress --- src/packages/frontend/nats/client.ts | 19 ++++--------------- src/packages/frontend/project_actions.ts | 24 ++++++++++++++++++++++++ src/packages/frontend/project_store.ts | 2 ++ src/packages/project/nats/open-files.ts | 2 +- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index d1affba8bd..b037e4cf76 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -350,12 +350,12 @@ export class NatsClient { openFiles.on("change", (entry) => { if (entry.deleted) { // if this file is opened here, close it. - ensureClosed({ project_id, path: entry.path, openFiles }); + setDeleted({ project_id, path: entry.path, openFiles }); } }); for (const entry of openFiles.getAll()) { if (entry.deleted) { - ensureClosed({ + setDeleted({ project_id, path: entry.path, openFiles, @@ -468,24 +468,13 @@ export class NatsClient { }; } -async function ensureClosed({ project_id, path, openFiles }) { +async function setDeleted({ project_id, path, openFiles }) { // console.log("ensureClosed", { path }); if (!redux.hasProjectStore(project_id)) { // console.log("ensureClosed: project not opened"); // file can't be opened if project isn't opened return; } - const store = redux.getProjectStore(project_id); - if (!store.get("open_files").has(path)) { - // console.log("ensureClosed: file not opened"); - return; - } - if (await store.get_listings().exists(path)) { - // console.log("ensureClosed: file exists on disk -- setting not deleted"); - openFiles.setNotDeleted(path); - return; - } - // console.log("ensureClosed: closing"); const actions = redux.getProjectActions(project_id); - actions.close_tab(path); + actions.setRecentlyDeleted(path); } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index e85de328c3..d200e34f27 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -3548,4 +3548,28 @@ export class ProjectActions extends Actions { change_history: true, }); }; + + setRecentlyDeleted = (path: string) => { + const store = this.get_store(); + if (store == null) return; + let recentlyDeletedFiles = store.get("recentlyDeletedFiles") ?? Set(); + recentlyDeletedFiles = recentlyDeletedFiles.add(path); + this.setState({ recentlyDeletedFiles }); + }; + + setNotDeleted = (path: string) => { + const store = this.get_store(); + if (store == null) return; + let recentlyDeletedFiles = store.get("recentlyDeletedFiles") ?? Set(); + recentlyDeletedFiles = recentlyDeletedFiles.delete(path); + this.setState({ recentlyDeletedFiles }); + (async () => { + try { + const o = await webapp_client.nats_client.openFiles(this.project_id); + o.setNotDeleted(path); + } catch (err) { + console.log("WARNING: issue undeleting file", err); + } + })(); + }; } diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 16e9b102dc..ab83ff6db7 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -113,6 +113,8 @@ export interface ProjectStoreState { file_listing_scroll_top?: number; new_filename?: string; ext_selection?: string; + // paths that were deleted + recentlyDeletedFiles?: immutable.Set; // Project Log project_log?: ProjectLogMap; diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 3d236fbfb0..ce0fdc4f0e 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -59,7 +59,7 @@ import "@cocalc/backend/nats"; const logger = getLogger("project:nats:open-files"); -const FILE_DELETION_CHECK_INTERVAL_MS = 2000; +const FILE_DELETION_CHECK_INTERVAL_MS = 3000; let openFiles: OpenFiles | null = null; let formatter: any = null; From cd3a551fa3663d069724435d26a8db5357ec5a5b Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 18 Feb 2025 17:48:06 +0000 Subject: [PATCH 230/281] nats: handling deleted files --- src/packages/frontend/chat/register.ts | 2 +- src/packages/frontend/client/file.ts | 14 ++--- .../frame-editors/frame-tree/leaf.tsx | 33 ++++------ src/packages/frontend/nats/client.ts | 33 +++++----- .../frontend/project/deleted-file.tsx | 33 +++++----- .../frontend/project/explorer/action-box.tsx | 8 ++- .../frontend/project/history/log-entry.tsx | 11 +++- .../frontend/project/history/types.ts | 2 + .../frontend/project/nats/listings.ts | 41 +++++------- src/packages/frontend/project/open-file.ts | 9 ++- .../frontend/project/page/content.tsx | 38 ++++++------ src/packages/frontend/project_actions.ts | 17 ++--- src/packages/frontend/project_store.ts | 62 +------------------ 13 files changed, 116 insertions(+), 187 deletions(-) diff --git a/src/packages/frontend/chat/register.ts b/src/packages/frontend/chat/register.ts index 59cd8a12d8..7ff96d84ca 100644 --- a/src/packages/frontend/chat/register.ts +++ b/src/packages/frontend/chat/register.ts @@ -23,7 +23,7 @@ export function initChat(project_id: string, path: string): ChatActions { if (startswith(path_split(path).tail, ".")) { // Sidechat being opened -- ensure chat isn't marked as deleted: - redux.getProjectStore(project_id)?.get_listings()?.undelete(path); + redux.getProjectActions(project_id)?.setNotDeleted(path); } const syncdb = webapp_client.sync_client.sync_db({ diff --git a/src/packages/frontend/client/file.ts b/src/packages/frontend/client/file.ts index 2acde86322..8000a8c999 100644 --- a/src/packages/frontend/client/file.ts +++ b/src/packages/frontend/client/file.ts @@ -18,7 +18,7 @@ export class FileClient { // Currently only used for testing and development in the console. public async syncdoc_history( string_id: string, - patches?: boolean + patches?: boolean, ): Promise { return ( await this.async_call({ @@ -33,15 +33,11 @@ export class FileClient { // Returns true if the given file in the given project is currently // marked as deleted. - public is_deleted(filename: string, project_id: string): boolean { + public is_deleted(path: string, project_id: string): boolean { return !!redux .getProjectStore(project_id) - ?.get_listings() - ?.isDeleted(filename); - } - - public undelete(filename: string, project_id: string): void { - redux.getProjectStore(project_id)?.get_listings()?.undelete(filename); + ?.get("recentlyDeletedPaths") + ?.get(path); } public set_deleted(_filename, _project_id): void { @@ -67,7 +63,7 @@ export class FileClient { } public async remove_blob_ttls( - uuids: string[] // list of sha1 hashes of blobs stored in the blobstore + uuids: string[], // list of sha1 hashes of blobs stored in the blobstore ) { if (uuids.length === 0) return; await this.async_call({ diff --git a/src/packages/frontend/frame-editors/frame-tree/leaf.tsx b/src/packages/frontend/frame-editors/frame-tree/leaf.tsx index 8ef2c7f829..e6f9d7926a 100644 --- a/src/packages/frontend/frame-editors/frame-tree/leaf.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/leaf.tsx @@ -7,20 +7,16 @@ import { CSS, React, Rendered, - useEffect, useRedux, - useState, + useTypedRedux, } from "@cocalc/frontend/app-framework"; import { ErrorDisplay, Loading } from "@cocalc/frontend/components"; import { AvailableFeatures } from "@cocalc/frontend/project_configuration"; import { Map, Set } from "immutable"; - import { AccountState } from "@cocalc/frontend/account/types"; import { Actions } from "../code-editor/actions"; import { EditorDescription, EditorState, NodeDesc } from "./types"; - import DeletedFile from "@cocalc/frontend/project/deleted-file"; -import { webapp_client } from "../../webapp-client"; const ERROR_STYLE: CSS = { maxWidth: "100%", @@ -110,37 +106,30 @@ export const FrameTreeLeaf: React.FC = React.memo( const read_only: boolean | undefined = useRedux(name, "read_only"); const cursors: Map | undefined = useRedux(name, "cursors"); + const recentlyDeletedPaths: Map | undefined = useTypedRedux( + { project_id }, + "recentlyDeletedPaths", + ); + const value: string | undefined = useRedux(name, "value"); const misspelled_words: Set | undefined = useRedux( name, - "misspelled_words" + "misspelled_words", ); const complete: Map | undefined = useRedux(name, "complete"); const is_loaded: boolean | undefined = useRedux(name, "is_loaded"); const error: string | undefined = useRedux(name, "error"); const gutter_markers: Map | undefined = useRedux( name, - "gutter_markers" + "gutter_markers", ); - const [isDeleted, setIsDeleted] = useState(false); - - useEffect(() => { - const p = desc.get("path"); - if (p == null) return; - if (webapp_client.file_client.is_deleted(p, project_id)) { - setIsDeleted(true); - } - }, [desc.get("path")]); - - if (isDeleted) { + if (recentlyDeletedPaths?.get(path)) { return ( { - setIsDeleted(false); - }} + time={recentlyDeletedPaths.get(path)!} /> ); } @@ -220,5 +209,5 @@ export const FrameTreeLeaf: React.FC = React.memo( {render_leaf()}
); - } + }, ); diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index b037e4cf76..e2d30c39f2 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -349,19 +349,19 @@ export class NatsClient { }); openFiles.on("change", (entry) => { if (entry.deleted) { - // if this file is opened here, close it. - setDeleted({ project_id, path: entry.path, openFiles }); + setDeleted({ project_id, path: entry.path, deleted: entry.deleted }); + } else { + setNotDeleted({ project_id, path: entry.path }); } }); - for (const entry of openFiles.getAll()) { - if (entry.deleted) { - setDeleted({ - project_id, - path: entry.path, - openFiles, - }); + const recentlyDeletedPaths: any = {}; + for (const { path, deleted } of openFiles.getAll()) { + if (deleted) { + recentlyDeletedPaths[path] = deleted; } } + const store = redux.getProjectStore(project_id); + store.setState({ recentlyDeletedPaths }); } return this.openFilesCache[project_id]!; }); @@ -468,13 +468,18 @@ export class NatsClient { }; } -async function setDeleted({ project_id, path, openFiles }) { - // console.log("ensureClosed", { path }); +function setDeleted({ project_id, path, deleted }) { + if (!redux.hasProjectStore(project_id)) { + return; + } + const actions = redux.getProjectActions(project_id); + actions.setRecentlyDeleted(path, deleted); +} + +function setNotDeleted({ project_id, path }) { if (!redux.hasProjectStore(project_id)) { - // console.log("ensureClosed: project not opened"); - // file can't be opened if project isn't opened return; } const actions = redux.getProjectActions(project_id); - actions.setRecentlyDeleted(path); + actions.setRecentlyDeleted(path, 0); } diff --git a/src/packages/frontend/project/deleted-file.tsx b/src/packages/frontend/project/deleted-file.tsx index b16e9dde9b..0b44f15007 100644 --- a/src/packages/frontend/project/deleted-file.tsx +++ b/src/packages/frontend/project/deleted-file.tsx @@ -4,19 +4,20 @@ */ import { Modal } from "antd"; -import { useCallback, useEffect, useState } from "react"; - +import { useCallback, useState } from "react"; import { redux } from "@cocalc/frontend/app-framework"; import useIsMountedRef from "@cocalc/frontend/app-framework/is-mounted-hook"; import { path_split } from "@cocalc/util/misc"; +import { TimeAgo } from "@cocalc/frontend/components"; +import { log_file_open } from "@cocalc/frontend/project/open-file"; interface Props { project_id: string; path: string; - onOpen?: Function; + time: number; } -export default function DeletedFile({ project_id, path, onOpen }: Props) { +export default function DeletedFile({ project_id, path, time }: Props) { const [open, setOpen] = useState(true); const isMountedRef = useIsMountedRef(); const { tail: filename } = path_split(path); @@ -24,27 +25,21 @@ export default function DeletedFile({ project_id, path, onOpen }: Props) { const openFile = useCallback(async () => { if (!isMountedRef.current) return; setOpen(false); - const store = redux.getProjectStore(project_id); - const listings = store.get_listings(); - await listings.undelete(path); - onOpen?.(); - }, []); - - useEffect(() => { - const store = redux.getProjectStore(project_id); - const listings = store.get_listings(); - (async () => { - if (await listings.exists(path)) { - openFile(); - } - })(); + const actions = redux.getProjectActions(project_id); + actions.setNotDeleted(path); + log_file_open(project_id, path, time); }, []); return (
+ The file "{filename}" was deleted or moved{" "} + . Open it anyways? +
+ } onOk={openFile} onCancel={() => { setOpen(false); diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index 144ab9f45a..0bc9251f59 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -115,9 +115,11 @@ export function ActionBox(props: ReactProps) { } function delete_click(): void { - props.actions.delete_files({ - paths: props.checked_files.toArray(), - }); + const paths = props.checked_files.toArray(); + for (const path of paths) { + props.actions.close_tab(path); + } + props.actions.delete_files({ paths }); props.actions.set_file_action(); props.actions.set_all_files_unchecked(); props.actions.fetch_directory_listing(); diff --git a/src/packages/frontend/project/history/log-entry.tsx b/src/packages/frontend/project/history/log-entry.tsx index 1f3e19e260..2890d44981 100644 --- a/src/packages/frontend/project/history/log-entry.tsx +++ b/src/packages/frontend/project/history/log-entry.tsx @@ -4,7 +4,6 @@ */ import { Space, Tooltip } from "antd"; import React from "react"; - import { Avatar } from "@cocalc/frontend/account/avatar/avatar"; import { Col, Grid, Row } from "@cocalc/frontend/antd-bootstrap"; import { @@ -168,6 +167,12 @@ export const LogEntry: React.FC = React.memo( } />{" "} + {event.deleted && ( + <> + {" "} + (file was deleted ) + + )} ); } @@ -205,10 +210,10 @@ export const LogEntry: React.FC = React.memo( ): JSX.Element { const envs = software_envs?.get("environments"); const prev: string = envs - ? envs.get(event.previous)?.get("title") ?? event.previous + ? (envs.get(event.previous)?.get("title") ?? event.previous) : intl.formatMessage(labels.loading); const next: string = envs - ? envs.get(event.next)?.get("title") ?? event.next + ? (envs.get(event.next)?.get("title") ?? event.next) : intl.formatMessage(labels.loading); return ( diff --git a/src/packages/frontend/project/history/types.ts b/src/packages/frontend/project/history/types.ts index babf84987f..32cbf50df4 100644 --- a/src/packages/frontend/project/history/types.ts +++ b/src/packages/frontend/project/history/types.ts @@ -177,6 +177,8 @@ export type OpenFile = { filename: string; time?: number; type?: string; + // if true, opening a file that was deleted + deleted?: number; }; export type ProjectControlEvent = { diff --git a/src/packages/frontend/project/nats/listings.ts b/src/packages/frontend/project/nats/listings.ts index 292bc84089..ff0f553a29 100644 --- a/src/packages/frontend/project/nats/listings.ts +++ b/src/packages/frontend/project/nats/listings.ts @@ -3,7 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ -import { TypedMap } from "@cocalc/frontend/app-framework"; +import { TypedMap, redux } from "@cocalc/frontend/app-framework"; import { webapp_client } from "@cocalc/frontend/webapp-client"; import { Listing } from "@cocalc/util/db-schema/listings"; import type { DirectoryListingEntry } from "@cocalc/util/types"; @@ -78,32 +78,14 @@ export class Listings extends EventEmitter { } return x.files; } - // TODO: trigger_start_project ? - return await this.api.getListing({ path, hidden: true }); - }; - - getDeleted = async (path: string): Promise | undefined> => { - if (this.listingsClient == null) { - throw Error("listings not ready"); - } - // TODO -- or not - return undefined; - }; - - undelete = async (filename: string): Promise => { - if (this.listingsClient == null) { - throw Error("listings not ready"); + if (trigger_start_project) { + if ( + !(await redux.getActions("projects").start_project(this.project_id)) + ) { + return; + } } - // TODO - }; - - // true or false if known deleted or not; undefined if don't know yet. - // TODO: technically we should check the all the - // deleted_file_variations... but that is really an edge case - // that probably doesn't matter much. - public isDeleted = (filename: string): boolean | undefined => { - // TODO - return false; + return await this.api.getListing({ path, hidden: true }); }; // Does a call to the project to directly determine whether or @@ -168,6 +150,13 @@ export class Listings extends EventEmitter { trigger_start_project?: boolean, ): Promise => { console.trace("getListingDirectly", { path }); + if (trigger_start_project) { + if ( + !(await redux.getActions("projects").start_project(this.project_id)) + ) { + throw Error("project not running"); + } + } // todo: trigger_start_project return await this.api.getListing({ path, hidden: true }); }; diff --git a/src/packages/frontend/project/open-file.ts b/src/packages/frontend/project/open-file.ts index dd89877a40..2eea3c364a 100644 --- a/src/packages/frontend/project/open-file.ts +++ b/src/packages/frontend/project/open-file.ts @@ -420,12 +420,16 @@ async function convert_sagenb_worksheet( const log_open_time: { [path: string]: { id: string; start: Date } } = {}; -export function log_file_open(project_id: string, path: string): void { +export function log_file_open( + project_id: string, + path: string, + deleted?: number, +): void { // Only do this if the file isn't // deleted, since if it *is* deleted, then user sees a dialog // and we only log the open if they select to recreate the file. // See https://github.com/sagemathinc/cocalc/issues/4720 - if (webapp_client.file_client.is_deleted(path, project_id)) { + if (!deleted && webapp_client.file_client.is_deleted(path, project_id)) { return; } @@ -435,6 +439,7 @@ export function log_file_open(project_id: string, path: string): void { event: "open", action: "open", filename: path, + deleted, }); // Save the log entry id, so it is possible to optionally diff --git a/src/packages/frontend/project/page/content.tsx b/src/packages/frontend/project/page/content.tsx index 15fbb8c4f6..ea10e73928 100644 --- a/src/packages/frontend/project/page/content.tsx +++ b/src/packages/frontend/project/page/content.tsx @@ -21,7 +21,6 @@ import { React, ReactDOM, redux, - useForceUpdate, useTypedRedux, } from "@cocalc/frontend/app-framework"; import { KioskModeBanner } from "@cocalc/frontend/app/kiosk-mode-banner"; @@ -40,12 +39,10 @@ import { Explorer } from "@cocalc/frontend/project/explorer"; import { ProjectLog } from "@cocalc/frontend/project/history"; import { ProjectInfo } from "@cocalc/frontend/project/info"; import { ProjectNew } from "@cocalc/frontend/project/new"; -import { log_file_open } from "@cocalc/frontend/project/open-file"; import { ProjectSearch } from "@cocalc/frontend/project/search/search"; import { ProjectServers } from "@cocalc/frontend/project/servers"; import { ProjectSettings } from "@cocalc/frontend/project/settings"; import { editor_id } from "@cocalc/frontend/project/utils"; -import { webapp_client } from "@cocalc/frontend/webapp-client"; import { hidden_meta_file } from "@cocalc/util/misc"; import { useProjectContext } from "../context"; import getAnchorTagComponent from "./anchor-tag-component"; @@ -103,6 +100,10 @@ const TabContent: React.FC = (props: TabContentProps) => { useTypedRedux({ project_id }, "open_files") ?? Map(); const fullscreen = useTypedRedux("page", "fullscreen"); const jupyterApiEnabled = useTypedRedux("customize", "jupyter_api_enabled"); + const recentlyDeletedPaths: Map | undefined = useTypedRedux( + { project_id }, + "recentlyDeletedPaths", + ); const path = useMemo(() => { if (tab_name.startsWith("editor-")) { @@ -185,6 +186,7 @@ const TabContent: React.FC = (props: TabContentProps) => { DEFAULT_CHAT_WIDTH } component={open_files.getIn([path, "component"]) ?? {}} + deleted={recentlyDeletedPaths?.get(path)} /> ); @@ -234,27 +236,23 @@ interface EditorContentProps { chat_width: number; chatState?: ChatState; component: { Editor?; redux_name?: string }; + // if deleted, when + deleted?: number; } -const EditorContent: React.FC = ( - props: EditorContentProps, -) => { - const { project_id, path, chat_width, is_visible, chatState, component } = - props; +const EditorContent: React.FC = ({ + deleted, + project_id, + path, + chat_width, + is_visible, + chatState, + component, +}: EditorContentProps) => { const editor_container_ref = useRef(null); - const force_update = useForceUpdate(); - if (webapp_client.file_client.is_deleted(path, project_id)) { - return ( - { - log_file_open(project_id, path); - force_update(); - }} - /> - ); + if (deleted) { + return ; } // Render this here, since it is used in multiple places below. diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index d200e34f27..0b1122aa40 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -3549,20 +3549,23 @@ export class ProjectActions extends Actions { }); }; - setRecentlyDeleted = (path: string) => { + // time = 0 to undelete + setRecentlyDeleted = (path: string, time: number) => { const store = this.get_store(); if (store == null) return; - let recentlyDeletedFiles = store.get("recentlyDeletedFiles") ?? Set(); - recentlyDeletedFiles = recentlyDeletedFiles.add(path); - this.setState({ recentlyDeletedFiles }); + let recentlyDeletedPaths = store.get("recentlyDeletedPaths") ?? Map(); + if (time == (recentlyDeletedPaths.get(path) ?? 0)) { + // already done + return; + } + recentlyDeletedPaths = recentlyDeletedPaths.set(path, time); + this.setState({ recentlyDeletedPaths }); }; setNotDeleted = (path: string) => { const store = this.get_store(); if (store == null) return; - let recentlyDeletedFiles = store.get("recentlyDeletedFiles") ?? Set(); - recentlyDeletedFiles = recentlyDeletedFiles.delete(path); - this.setState({ recentlyDeletedFiles }); + this.setRecentlyDeleted(path, 0); (async () => { try { const o = await webapp_client.nats_client.openFiles(this.project_id); diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index ab83ff6db7..26179a0ec1 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -17,7 +17,6 @@ if (typeof window !== "undefined" && window !== null) { import * as immutable from "immutable"; -import { alert_message } from "@cocalc/frontend/alerts"; import { AppRedux, project_redux_name, @@ -38,7 +37,6 @@ import { isMainConfiguration, ProjectConfiguration, } from "@cocalc/frontend/project_configuration"; -import { deleted_file_variations } from "@cocalc/util/delete-files"; import * as misc from "@cocalc/util/misc"; import { compute_file_masks } from "./project/explorer/compute-file-masks"; import { DirectoryListing } from "./project/explorer/types"; @@ -114,7 +112,7 @@ export interface ProjectStoreState { new_filename?: string; ext_selection?: string; // paths that were deleted - recentlyDeletedFiles?: immutable.Set; + recentlyDeletedPaths?: immutable.Map; // Project Log project_log?: ProjectLogMap; @@ -583,70 +581,12 @@ export class ProjectStore extends Store { return this.getIn(["open_files", path, "component"])?.Editor != null; } - private close_deleted_file(path: string): void { - const cur = this.get("current_path"); - if (path == cur || misc.startswith(cur, path + "/")) { - // we are deleting the current directory, so let's cd to HOME. - const actions = redux.getProjectActions(this.project_id); - if (actions != null) { - actions.set_current_path(""); - } - } - const all_paths = deleted_file_variations(path); - const open_files = this.get("open_files"); - if (open_files == null) { - return; - } - for (const file of this.get("open_files").keys()) { - if (all_paths.indexOf(file) != -1 || misc.startswith(file, path + "/")) { - if (!this.has_file_been_viewed(file)) { - // Hasn't even been viewed yet; when user clicks on the tab - // they get a dialog to undelete the file. - continue; - } - const actions = redux.getProjectActions(this.project_id); - if (actions != null) { - actions.close_tab(file); - alert_message({ - type: "info", - message: `Closing '${file}' since it was deleted or moved.`, - }); - } - } else { - const actions: any = redux.getEditorActions(this.project_id, file); - if (actions?.close_frames_with_path != null) { - // close subframes with given path. - if (actions.close_frames_with_path(path)) { - alert_message({ - type: "info", - message: `Closed '${path}' in '${file}' since it was deleted or moved.`, - }); - } - } - } - } - } - public get_listings(compute_server_id: number | null = null): Listings { const computeServerId = compute_server_id ?? this.get("compute_server_id"); if (this.listings[computeServerId] == null) { const listingsTable = listings(this.project_id, computeServerId); this.listings[computeServerId] = listingsTable; listingsTable.watch(this.get("current_path") ?? "", true); - listingsTable.on("deleted", async (paths) => { - for (const path of paths) { - if (this.listings[0] == null) return; // shouldn't happen - const deleted = await listingsTable.getDeleted(path); - if (deleted != null) { - for (let filename of deleted) { - if (path != "") { - filename = path + "/" + filename; - } - this.close_deleted_file(filename); - } - } - } - }); listingsTable.on("change", async (paths) => { let directory_listings_for_server = this.getIn(["directory_listings", computeServerId]) ?? From b7c2ca2173eb901d6a716d57929937bfb6e87db1 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 18 Feb 2025 19:06:51 +0000 Subject: [PATCH 231/281] nats: starting project and initial listing -- improve --- .../frontend/project/nats/listings.ts | 86 +++++++++++++++++-- src/packages/nats/service/listings.ts | 27 ++++-- 2 files changed, 98 insertions(+), 15 deletions(-) diff --git a/src/packages/frontend/project/nats/listings.ts b/src/packages/frontend/project/nats/listings.ts index ff0f553a29..80fcd20173 100644 --- a/src/packages/frontend/project/nats/listings.ts +++ b/src/packages/frontend/project/nats/listings.ts @@ -16,6 +16,8 @@ import { type ListingsApi, MIN_INTEREST_INTERVAL_MS, } from "@cocalc/nats/service/listings"; +import { delay } from "awaiting"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; export const WATCH_THROTTLE_MS = MIN_INTEREST_INTERVAL_MS; @@ -40,11 +42,48 @@ export class Listings extends EventEmitter { this.init(); } - private init = async () => { - this.listingsClient = await listingsClient({ - project_id: this.project_id, - compute_server_id: this.compute_server_id, - }); + private createClient = async () => { + let d = 3000; + const MAX_DELAY_MS = 15000; + while (this.state != "closed") { + try { + this.listingsClient = await listingsClient({ + project_id: this.project_id, + compute_server_id: this.compute_server_id, + }); + // success! + return; + } catch (err) { + // console.log("creating listings client failed", err); + if (err.code == "PERMISSIONS_VIOLATION") { + try { +// console.log( +// `request update of our credentials to include ${this.project_id}, then try again`, +// ); + await webapp_client.nats_client.hub.projects.addProjectPermission({ + project_id: this.project_id, + }); + continue; + } catch (err) { + // console.log("updating permissions failed", err); + d = Math.max(7000, d); + } + } + } + if (this.state == ("closed" as State)) return; + d = Math.min(MAX_DELAY_MS, d * 1.3); + await delay(d); + } + }; + + private init = reuseInFlight(async () => { + let start = Date.now(); + await this.createClient(); + // console.log("createClient finished in ", Date.now() - start, "ms"); + if (this.state == "closed") return; + if (this.listingsClient == null) { + throw Error("bug"); + } this.listingsClient.on("change", (path) => { this.emit("change", [path]); }); @@ -52,13 +91,44 @@ export class Listings extends EventEmitter { this.emit("change", Object.keys(this.listingsClient.getAll())); // [ ] TODO: delete event for deleted paths this.setState("ready"); - }; + }); // Watch directory for changes. watch = async (path: string, force?): Promise => { + if (this.state == "closed") { + return; + } + if (this.state != "ready") { + await this.init(); + } + if (this.state != "ready") { + // failed forever or closed explicitly so don't care + return; + } + if (this.listingsClient == null) { + throw Error("listings not ready"); + } try { - await this.listingsClient?.watch(path, force); - } catch {} + await this.listingsClient.watch(path, force); + } catch (err) { + if (err.code == "503") { + // The listings service isn't running in the project right now, + // e.g., maybe the project isn't running at all. + // So watch is a no-op, as it does nothing when listing + // server doesn't exist. So we at least wait for a while + // e.g., maybe project is tarting, then try once more. + await this.listingsClient.api.nats.waitFor(); + try { + await this.listingsClient.watch(path, force); + } catch (err) { + if (err.code != "503") { + throw err; + } + } + } else { + throw err; + } + } }; get = async ( diff --git a/src/packages/nats/service/listings.ts b/src/packages/nats/service/listings.ts index 1fc277df58..a43f1da2fe 100644 --- a/src/packages/nats/service/listings.ts +++ b/src/packages/nats/service/listings.ts @@ -109,7 +109,7 @@ export async function getListingsTimesKV( export class ListingsClient extends EventEmitter { options: { project_id: string; compute_server_id: number }; - api: ListingsApi; + api: Awaited>; times?: DKV; listings?: DKV; @@ -125,10 +125,19 @@ export class ListingsClient extends EventEmitter { } init = async () => { - this.api = createListingsApiClient(this.options); - this.times = await getListingsTimesKV(this.options); - this.listings = await getListingsKV(this.options); - this.listings.on("change", ({ key: path }) => this.emit("change", path)); + try { + this.api = createListingsApiClient(this.options); + this.times = await getListingsTimesKV(this.options); + this.listings = await getListingsKV(this.options); + this.listings.on("change", this.handleListingsChange); + } catch (err) { + this.close(); + throw err; + } + }; + + handleListingsChange = ({ key: path }) => { + this.emit("change", path); }; get = (path: string): Listing | undefined => { @@ -146,10 +155,14 @@ export class ListingsClient extends EventEmitter { }; close = () => { + this.removeAllListeners(); this.times?.close(); delete this.times; - this.listings?.close(); - delete this.listings; + if (this.listings != null) { + this.listings.removeListener("change", this.handleListingsChange); + this.listings.close(); + delete this.listings; + } }; watch = async (path, force = false) => { From d1bb47eddcde98dbe35b5792f2a6a8ee2b475e72 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 19 Feb 2025 00:57:41 +0000 Subject: [PATCH 232/281] nats compute servers: change frontend to use new nats based manager --- src/packages/frontend/compute/manager.ts | 165 ++--------------- .../jupyter-editor/jupyter-actions.ts | 2 +- src/packages/frontend/nats/client.ts | 10 ++ .../frontend/project/nats/listings.ts | 2 +- src/packages/nats/compute/README.md | 2 + src/packages/nats/compute/manager.ts | 170 ++++++++++++++++++ src/packages/nats/package.json | 1 + src/packages/nats/sync/open-files.ts | 5 +- 8 files changed, 206 insertions(+), 151 deletions(-) create mode 100644 src/packages/nats/compute/README.md create mode 100644 src/packages/nats/compute/manager.ts diff --git a/src/packages/frontend/compute/manager.ts b/src/packages/frontend/compute/manager.ts index 36ba37f46d..5344c07cc4 100644 --- a/src/packages/frontend/compute/manager.ts +++ b/src/packages/frontend/compute/manager.ts @@ -6,159 +6,30 @@ are available and how they are used for a given project. When doing dev from the browser console, do: -cc.client.project_client.computeServers('...project_id...') + cc.client.project_client.computeServers(cc.current().project_id) */ -import { SYNCDB_PARAMS, decodeUUIDtoNum } from "@cocalc/util/compute/manager"; -import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { redux } from "@cocalc/frontend/app-framework"; -import debug from "debug"; -import { once } from "@cocalc/util/async-utils"; -import { EventEmitter } from "events"; -import { excludeFromComputeServer } from "@cocalc/frontend/file-associations"; - -const log = debug("cocalc:frontend:compute:manager"); - -export class ComputeServersManager extends EventEmitter { - private sync_db; - private project_id; - - constructor(project_id: string) { - super(); - this.project_id = project_id; - this.sync_db = webapp_client.sync_db({ - project_id, - ...SYNCDB_PARAMS, - }); - this.sync_db.on("change", () => { - this.emit("change"); - }); - // It's reasonable to have many clients, e.g., one for each open file - this.setMaxListeners(100); - log("created", this.project_id); - } - - waitUntilReady = async () => { - const { sync_db } = this; - if (sync_db.get_state() == "init") { - // make sure project is running - redux.getActions("projects").start_project(this.project_id); - - // now wait for syncdb to be ready - await once(sync_db, "ready"); - } - if (sync_db.get_state() != "ready") { - throw Error("syncdb not ready"); - } - }; - - close = () => { - delete computeServerManagerCache[this.project_id]; - this.sync_db.close(); - }; - - // save the current state to the backend. This is critical to do, e.g., before - // opening a file and after calling connectComputeServerToPath, since otherwise - // the project doesn't even know that the file should open on the compute server - // until after it has opened it, which is disconcerting and not efficient (but - // does mostly work, though it is intentionally making things really hard on ourselves). - save = async () => { - await this.sync_db.save(); - }; - - getComputeServers = () => { - const servers = {}; - const cursors = this.sync_db.get_cursors({ excludeSelf: "never" }).toJS(); - for (const client_id in cursors) { - const server = cursors[client_id]; - servers[decodeUUIDtoNum(client_id)] = { - time: server.time, - ...server.locs[0], - }; - } - return servers; - }; - - // Call this if you want the compute server with given id to - // connect and handle being the server for the given path. - connectComputeServerToPath = ({ id, path }: { id: number; path: string }) => { - if (id == 0) { - this.disconnectComputeServer({ path }); - return; - } - assertSupportedPath(path); - this.sync_db.set({ id, path, open: true }); - this.sync_db.commit(); - }; - - // Call this if you want no compute servers to provide the backend server - // for given path. - disconnectComputeServer = ({ path }: { path: string }) => { - this.sync_db.delete({ path }); - this.sync_db.commit(); - }; - - // For interactive debugging -- display in the console how things are configured. - showStatus = () => { - console.log(JSON.stringify(this.sync_db.get().toJS(), undefined, 2)); - }; - - // Returns the explicitly set server id for the given - // path, if one is set. Otherwise, return undefined - // if nothing is explicitly set for this path. - getServerIdForPath = async (path: string): Promise => { - await this.waitUntilReady(); - const { sync_db } = this; - return sync_db.get_one({ path })?.get("id"); - }; - - // Get the server ids (as a map) for every file and every directory contained in path. - // NOTE/TODO: this just does a linear search through all paths with a server id; nothing clever. - getServerIdForSubtree = async ( - path: string, - ): Promise<{ [path: string]: number }> => { - const { sync_db } = this; - if (sync_db.get_state() == "init") { - await once(sync_db, "ready"); - } - if (sync_db.get_state() != "ready") { - throw Error("syncdb not ready"); - } - const x = sync_db.get(); - const v: { [path: string]: number } = {}; - if (x == null) { - return v; - } - const slash = path.endsWith("/") ? path : path + "/"; - for (const y of x) { - const p = y.get("path"); - if (p == path || p.startsWith(slash)) { - v[p] = y.get("id"); - } - } - return v; - }; -} - -function assertSupportedPath(path: string) { - if (excludeFromComputeServer(path)) { - throw Error( - `Opening '${path}' on a compute server is not yet supported -- copy it to the project and open it there instead`, - ); - } -} +import { + computeServerManager, + type ComputeServerManager, +} from "@cocalc/nats/compute/manager"; const computeServerManagerCache: { - [project_id: string]: ComputeServersManager; + [project_id: string]: ComputeServerManager; } = {}; -export const computeServers = (project_id: string) => { +// very simple cache with no ref counting or anything. +// close a manager only when closing the project. +export default function computeServers( + project_id: string, +): ComputeServerManager { if (computeServerManagerCache[project_id]) { return computeServerManagerCache[project_id]; } - const m = new ComputeServersManager(project_id); - computeServerManagerCache[project_id] = m; - return m; -}; - -export default computeServers; + const M = computeServerManager({ project_id }); + computeServerManagerCache[project_id] = M; + M.on("close", () => { + delete computeServerManagerCache[project_id]; + }); + return M; +} diff --git a/src/packages/frontend/frame-editors/jupyter-editor/jupyter-actions.ts b/src/packages/frontend/frame-editors/jupyter-editor/jupyter-actions.ts index 9050b1bb02..e0ebe76089 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/jupyter-actions.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/jupyter-actions.ts @@ -37,7 +37,7 @@ export function create_jupyter_actions( // Ensure meta_file isn't marked as deleted, which would block // opening the syncdb, which is clearly not the user's intention // at this point (since we're opening the ipynb file). - redux.getProjectStore(project_id)?.get_listings()?.undelete(syncdb_path); + redux.getProjectActions(project_id)?.setNotDeleted(syncdb_path); const syncdb = new_syncdb({ ...SYNCDB_OPTIONS, diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index e2d30c39f2..5f012f174a 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -42,6 +42,10 @@ import type { CreateNatsServiceFunction, } from "@cocalc/nats/service"; import { listingsClient } from "@cocalc/nats/service/listings"; +import { + computeServerManager, + type Options as ComputeServerManagerOptions, +} from "@cocalc/nats/compute/manager"; export class NatsClient { client: WebappClient; @@ -466,6 +470,12 @@ export class NatsClient { }) => { return await listingsClient(opts); }; + + computeServerManager = async (options: ComputeServerManagerOptions) => { + const M = computeServerManager(options); + await M.init(); + return M; + }; } function setDeleted({ project_id, path, deleted }) { diff --git a/src/packages/frontend/project/nats/listings.ts b/src/packages/frontend/project/nats/listings.ts index 80fcd20173..80ed8ef436 100644 --- a/src/packages/frontend/project/nats/listings.ts +++ b/src/packages/frontend/project/nats/listings.ts @@ -77,7 +77,7 @@ export class Listings extends EventEmitter { }; private init = reuseInFlight(async () => { - let start = Date.now(); + //let start = Date.now(); await this.createClient(); // console.log("createClient finished in ", Date.now() - start, "ms"); if (this.state == "closed") return; diff --git a/src/packages/nats/compute/README.md b/src/packages/nats/compute/README.md new file mode 100644 index 0000000000..e7d46fc174 --- /dev/null +++ b/src/packages/nats/compute/README.md @@ -0,0 +1,2 @@ +Code related to compute servers + diff --git a/src/packages/nats/compute/manager.ts b/src/packages/nats/compute/manager.ts new file mode 100644 index 0000000000..5bacb621c2 --- /dev/null +++ b/src/packages/nats/compute/manager.ts @@ -0,0 +1,170 @@ +/* + +Used mainly from a browser client frontend to manage what compute server +is used to edit a given file. + +Access this in the browser for the project you have open: + +> m = await cc.client.nats_client.computeServerManager({project_id:cc.current().project_id}) + +*/ + +import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { EventEmitter } from "events"; + +type State = "init" | "connected" | "closed"; + +export interface Info { + // compute server where this path should be opened + id: number; +} + +export interface Options { + project_id: string; + noAutosave?: boolean; + noCache?: boolean; +} + +export function computeServerManager(options: Options) { + const M = new ComputeServerManager(options); + M.init(); + return M; +} + +export class ComputeServerManager extends EventEmitter { + private dkv?: DKV; + private options: Options; + private state: State = "init"; + + constructor(options: Options) { + super(); + this.options = options; + } + + waitUntilReady = async () => { + if (this.state == "closed") { + throw Error("manager is closed"); + } else if (this.state == "connected") { + return; + } + await this.init(); + }; + + save = async () => { + await this.dkv?.save(); + }; + + init = reuseInFlight(async () => { + try { + const d = await dkv({ + name: "compute-server-manager", + ...this.options, + }); + this.dkv = d; + d.on("change", this.handleChange); + this.setState("connected"); + } catch (err) { + console.warn("failed to create compute server manager", err); + this.close(); + throw err; + } + }); + + private handleChange = ({ key: path, value, prev }) => { + this.emit("change", { + path, + id: value?.id, + prev_id: prev?.id, + }); + }; + + close = () => { + console.warn("closing a compute server manager"); + if (this.dkv != null) { + this.dkv.removeListener("change", this.handleChange); + this.dkv.close(); + delete this.dkv; + } + this.setState("closed"); + this.removeAllListeners(); + }; + + private setState = (state: State) => { + this.state = state; + this.emit(state); + }; + + private getDkv = () => { + if (this.dkv == null) { + throw Error(`compute server not initialized -- in state '${this.state}'`); + } + return this.dkv; + }; + + getAll = async () => { + await this.waitUntilReady(); + return this.getDkv().getAll(); + }; + + // Call this if you want the compute server with given id to + // connect and handle being the server for the given path. + connectComputeServerToPath = async ({ + path, + id, + }: { + path: string; + id: number; + }) => { + await this.waitUntilReady(); + const kv = this.getDkv(); + if (!id) { + kv.delete(path); + return; + } + kv.set(path, { id }); + }; + + set = async (path, id) => { + await this.connectComputeServerToPath({ path, id }); + }; + + // Call this if you want no compute servers to provide the backend server + // for given path. + disconnectComputeServer = async ({ path }: { path: string }) => { + await this.waitUntilReady(); + this.getDkv().delete(path); + }; + + delete = async (path) => { + await this.disconnectComputeServer({ path }); + }; + + // Returns the explicitly set server id for the given + // path, if one is set. Otherwise, return undefined + // if nothing is explicitly set for this path (i.e., usually means home base). + getServerIdForPath = async (path: string): Promise => { + await this.waitUntilReady(); + return this.getDkv().get(path)?.id; + }; + + get = async (path) => this.getServerIdForPath(path); + + // Get the server ids (as a map) for every file and every directory contained in path. + // NOTE/TODO: this just does a linear search through all paths with a server id; nothing clever. + getServerIdForSubtree = async ( + path: string, + ): Promise<{ [path: string]: number }> => { + await this.waitUntilReady(); + const kv = this.getDkv(); + const v: { [path: string]: number } = {}; + const slash = path.endsWith("/") ? path : path + "/"; + const x = kv.getAll(); + for (const p in x) { + if (p == path || p.startsWith(slash)) { + v[p] = x[p].id; + } + } + return v; + }; +} diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 7f0519fb27..2028704591 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -6,6 +6,7 @@ "./sync/*": "./dist/sync/*.js", "./hub-api": "./dist/hub-api/index.js", "./hub-api/*": "./dist/hub-api/*.js", + "./compute/*": "./dist/compute/*.js", "./service": "./dist/service/index.js", "./project-api": "./dist/project-api/index.js", "./browser-api": "./dist/browser-api/index.js", diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 297bb08ece..3b99d6c503 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -34,8 +34,9 @@ import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; import { nanos } from "@cocalc/nats/util"; import { EventEmitter } from "events"; -// 1 day -const MAX_AGE_MS = 1000 * 60 * 60 * 24; +// info about interest in open files (and also what was explicitly deleted) older +// than this is automatically purged. +const MAX_AGE_MS = 7 * (1000 * 60 * 60 * 24); export interface Entry { // path to file relative to HOME From 2f539f0e9ca5907c6a3ac7db2e7b44f6930cfdf0 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 19 Feb 2025 01:42:46 +0000 Subject: [PATCH 233/281] nats compute servers -- make project open-files use the new manager in the backend to decide whether to open/close file --- src/packages/frontend/compute/manager.ts | 2 +- src/packages/nats/compute/manager.ts | 46 +++++++++++++----------- src/packages/project/nats/open-files.ts | 41 ++++++++++++++++++--- 3 files changed, 63 insertions(+), 26 deletions(-) diff --git a/src/packages/frontend/compute/manager.ts b/src/packages/frontend/compute/manager.ts index 5344c07cc4..25cc99824a 100644 --- a/src/packages/frontend/compute/manager.ts +++ b/src/packages/frontend/compute/manager.ts @@ -28,7 +28,7 @@ export default function computeServers( } const M = computeServerManager({ project_id }); computeServerManagerCache[project_id] = M; - M.on("close", () => { + M.on("closed", () => { delete computeServerManagerCache[project_id]; }); return M; diff --git a/src/packages/nats/compute/manager.ts b/src/packages/nats/compute/manager.ts index 5bacb621c2..b7e36ab7d7 100644 --- a/src/packages/nats/compute/manager.ts +++ b/src/packages/nats/compute/manager.ts @@ -40,6 +40,8 @@ export class ComputeServerManager extends EventEmitter { constructor(options: Options) { super(); this.options = options; + // It's reasonable to have many clients, e.g., one for each open file + this.setMaxListeners(100); } waitUntilReady = async () => { @@ -102,11 +104,30 @@ export class ComputeServerManager extends EventEmitter { return this.dkv; }; - getAll = async () => { - await this.waitUntilReady(); + // Modern sync API: used in backend. + + set = (path, id) => { + const kv = this.getDkv(); + if (!id) { + kv.delete(path); + return; + } + kv.set(path, { id }); + }; + + delete = (path) => { + this.getDkv().delete(path); + }; + + get = (path) => this.getDkv().get(path)?.id; + + getAll = () => { return this.getDkv().getAll(); }; + // Async API that doesn't assume manager has been initialized, with + // very long names. Used in the frontend. + // Call this if you want the compute server with given id to // connect and handle being the server for the given path. connectComputeServerToPath = async ({ @@ -117,27 +138,14 @@ export class ComputeServerManager extends EventEmitter { id: number; }) => { await this.waitUntilReady(); - const kv = this.getDkv(); - if (!id) { - kv.delete(path); - return; - } - kv.set(path, { id }); - }; - - set = async (path, id) => { - await this.connectComputeServerToPath({ path, id }); + this.set(path, id); }; // Call this if you want no compute servers to provide the backend server // for given path. disconnectComputeServer = async ({ path }: { path: string }) => { await this.waitUntilReady(); - this.getDkv().delete(path); - }; - - delete = async (path) => { - await this.disconnectComputeServer({ path }); + this.delete(path); }; // Returns the explicitly set server id for the given @@ -145,11 +153,9 @@ export class ComputeServerManager extends EventEmitter { // if nothing is explicitly set for this path (i.e., usually means home base). getServerIdForPath = async (path: string): Promise => { await this.waitUntilReady(); - return this.getDkv().get(path)?.id; + return this.get(path); }; - get = async (path) => this.getServerIdForPath(path); - // Get the server ids (as a map) for every file and every directory contained in path. // NOTE/TODO: this just does a linear search through all paths with a server id; nothing clever. getServerIdForSubtree = async ( diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index ce0fdc4f0e..fc7363dd96 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -16,7 +16,7 @@ Set env variables as in a project (see api/index.ts ), then in nodejs: DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:nats:* node > x = await require("@cocalc/project/nats/open-files").init(); Object.keys(x) -[ 'openFiles', 'openDocs', 'formatter', 'terminate' ] +[ 'openFiles', 'openDocs', 'formatter', 'terminate', 'computeServers' ] > x.openFiles.getAll(); @@ -35,7 +35,7 @@ import { } from "@cocalc/project/nats/sync"; import { getSyncDocType } from "@cocalc/nats/sync/syncdoc-info"; import { NATS_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/nats"; -import { /*compute_server_id,*/ project_id } from "@cocalc/project/data"; +import { compute_server_id, project_id } from "@cocalc/project/data"; import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; import { getClient } from "@cocalc/project/client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; @@ -53,6 +53,10 @@ import { exists } from "@cocalc/backend/misc/async-utils-node"; import { map as awaitMap } from "awaiting"; import { unlink } from "fs/promises"; import { join } from "path"; +import { + computeServerManager, + ComputeServerManager, +} from "@cocalc/nats/compute/manager"; // ensure nats connection stuff is initialized import "@cocalc/backend/nats"; @@ -64,12 +68,16 @@ const FILE_DELETION_CHECK_INTERVAL_MS = 3000; let openFiles: OpenFiles | null = null; let formatter: any = null; const openDocs: { [path: string]: SyncDoc | NatsService } = {}; +let computeServers: ComputeServerManager | null = null; export async function init() { logger.debug("init"); openFiles = await createOpenFiles(); + computeServers = computeServerManager({ project_id }); + await computeServers.init(); + // initialize for (const entry of openFiles.getAll()) { handleChange(entry); @@ -89,8 +97,8 @@ export async function init() { formatter = await createFormatterService({ openSyncDocs: openDocs }); - // usefule for development - return { openFiles, openDocs, formatter, terminate }; + // useful for development + return { openFiles, openDocs, formatter, terminate, computeServers }; } export function terminate() { @@ -103,16 +111,33 @@ export function terminate() { formatter?.close(); formatter = null; + + computeServers?.close(); + computeServers = null; } function getCutoff() { return new Date(Date.now() - 2.5 * NATS_OPEN_FILE_TOUCH_INTERVAL); } +function computeServerId(path: string): number { + return computeServers?.get(path) ?? 0; +} + async function handleChange({ path, open, time, deleted }: OpenFileEntry) { - logger.debug("handleChange", { path, open, time, deleted }); + const id = computeServerId(path); + logger.debug("handleChange", { path, open, time, deleted, id }); const syncDoc = openDocs[path]; const isOpenHere = syncDoc != null; + + if (id != compute_server_id) { + // only thing we should do is close it if it is open. + if (isOpenHere) { + await closeDoc(path); + } + return; + } + if (deleted) { if (await exists(path)) { // it's back @@ -196,6 +221,12 @@ async function checkForFileDeletion(path: string) { if (openFiles == null) { return; } + const id = computeServerId(path); + if (id != compute_server_id) { + // not our concern + return; + } + if (path.endsWith(".term")) { // term files are exempt -- we don't save data in them and often // don't actually make the hidden ones for each frame in the From 921b1aa9937257490766f81c1bd3551de9d9e563 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 19 Feb 2025 04:56:50 +0000 Subject: [PATCH 234/281] nats compute servers: improve file and terminal switching --- .../terminal-editor/connected-terminal.ts | 5 +++ src/packages/nats/service/service.ts | 9 ++++- src/packages/project/nats/open-files.ts | 35 ++++++++++++------- src/packages/project/nats/terminal.ts | 9 +++-- 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index e63bb5a9ce..015f155f95 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -271,6 +271,11 @@ export class Terminal { connect = async () => { try { + if (this.conn != null) { + this.conn.removeListener("close", this.connect); // avoid infinite loop + this.conn.close(); + delete this.conn; + } this.ignore_terminal_data = true; this.set_connection_status("connecting"); await this.configureComputeServerId(); diff --git a/src/packages/nats/service/service.ts b/src/packages/nats/service/service.ts index 7d5d2d65d7..947800bc40 100644 --- a/src/packages/nats/service/service.ts +++ b/src/packages/nats/service/service.ts @@ -21,6 +21,7 @@ import { sha1, trunc_middle } from "@cocalc/util/misc"; import { getEnv } from "@cocalc/nats/client"; import { randomId } from "@cocalc/nats/names"; import { delay } from "awaiting"; +import { EventEmitter } from "events"; const DEFAULT_TIMEOUT = 5000; @@ -151,12 +152,13 @@ export interface Options extends ServiceDescription { handler: (mesg) => Promise; } -export class NatsService { +export class NatsService extends EventEmitter { private options: Options; private subject: string; private api?; constructor(options: Options) { + super(); this.options = options; this.subject = serviceSubject(options); } @@ -193,6 +195,11 @@ export class NatsService { }; close = () => { + if (!this.subject) { + return; + } + this.emit("close"); + this.removeAllListeners(); this.api?.stop(); // @ts-ignore delete this.subject; diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index fc7363dd96..b880068177 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -77,6 +77,17 @@ export async function init() { computeServers = computeServerManager({ project_id }); await computeServers.init(); + computeServers.on("change", async ({ path, id }) => { + if (openFiles == null) { + return; + } + const entry = openFiles?.get(path); + if (entry != null) { + await handleChange({ ...entry, path, id }); + } else { + await closeDoc(path); + } + }); // initialize for (const entry of openFiles.getAll()) { @@ -124,8 +135,16 @@ function computeServerId(path: string): number { return computeServers?.get(path) ?? 0; } -async function handleChange({ path, open, time, deleted }: OpenFileEntry) { - const id = computeServerId(path); +async function handleChange({ + path, + open, + time, + deleted, + id, +}: OpenFileEntry & { id?: number }) { + if (id == null) { + id = computeServerId(path); + } logger.debug("handleChange", { path, open, time, deleted, id }); const syncDoc = openDocs[path]; const isOpenHere = syncDoc != null; @@ -149,17 +168,7 @@ async function handleChange({ path, open, time, deleted }: OpenFileEntry) { return; } } - // TODO: need another table with compute server mappings - // const id = 0; // todo - // if (id != compute_server_id) { - // if (isOpenHere) { - // // close it here - // logger.debug("handleChange: closing", { path }); - // closeDoc(path); - // } - // // no further responsibility - // return; - // } + if (!open) { if (isOpenHere) { logger.debug("handleChange: closing", { path }); diff --git a/src/packages/project/nats/terminal.ts b/src/packages/project/nats/terminal.ts index a4dce047d6..a1cb4977f9 100644 --- a/src/packages/project/nats/terminal.ts +++ b/src/packages/project/nats/terminal.ts @@ -94,7 +94,12 @@ export async function createTerminalService(path: string) { }, }; - return await createTerminalServer({ path, project_id, impl }); + const server = await createTerminalServer({ path, project_id, impl }); + server.on("close", () => { + sessions[path]?.close(); + delete sessions[path]; + }); + return server; } function closeTerminal(path: string) { @@ -288,7 +293,7 @@ class Session { // tell browsers about out new size await this.browserApi.size({ rows, cols }); } catch (err) { - logger.debug("terminal channel -- WARNING: unable to resize term", err); + logger.debug(`WARNING: unable to resize term: ${err}`); } }; From f2dfc8e64822095203e1fe90e85316eb14dd9aa5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 19 Feb 2025 14:23:10 +0000 Subject: [PATCH 235/281] nats: workaround typescript issue --- src/packages/project/nats/open-files.ts | 12 ++++++++++++ src/packages/sync/table/synctable.ts | 16 ++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index b880068177..b10fecf93d 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -26,6 +26,18 @@ DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:nats:* node // now you can directly work with the syncdoc for a given file, // but from the perspective of the project, not the browser! +COMPUTE SERVER: + +To simulate a compute server, do exactly as above, but also set the environment +variable COMPUTE_SERVER_ID to the *global* (not project specific) id of the compute +server: + + COMPUTE_SERVER_ID=84 node + +In this case, you aso don't need to use the terminate command if the compute +server isn't actually running. To terminate a compute server open files service though: + + (TODO) */ import { diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts index 9768948ac6..b0bef4df91 100644 --- a/src/packages/sync/table/synctable.ts +++ b/src/packages/sync/table/synctable.ts @@ -800,16 +800,20 @@ export class SyncTable extends EventEmitter { }; } + // awkward code due to typescript weirdness using both + // NatsChangefeed and Changefeed types (for unit testing). private init_changefeed_handlers(): void { - if (this.changefeed == null) return; - this.changefeed.on("update", this.changefeed_on_update); - this.changefeed.on("close", this.changefeed_on_close); + const c = this.changefeed as EventEmitter | null; + if (c == null) return; + c.on("update", this.changefeed_on_update); + c.on("close", this.changefeed_on_close); } private remove_changefeed_handlers(): void { - if (this.changefeed == null) return; - this.changefeed.removeListener("update", this.changefeed_on_update); - this.changefeed.removeListener("close", this.changefeed_on_close); + const c = this.changefeed as EventEmitter | null; + if (c == null) return; + c.removeListener("update", this.changefeed_on_update); + c.removeListener("close", this.changefeed_on_close); } private changefeed_on_update(change): void { From ed4433dc33e23a6340e5a1a130c4d6b7ba941a6a Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 19 Feb 2025 15:19:23 +0000 Subject: [PATCH 236/281] nats project connection: automatically reconnect until success --- src/packages/project/nats/connection.ts | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/packages/project/nats/connection.ts b/src/packages/project/nats/connection.ts index c5095b41cb..83bada9442 100644 --- a/src/packages/project/nats/connection.ts +++ b/src/packages/project/nats/connection.ts @@ -3,24 +3,35 @@ import { connect, jwtAuthenticator } from "nats"; import { CONNECT_OPTIONS } from "@cocalc/util/nats"; import { inboxPrefix as getInboxPrefix } from "@cocalc/nats/names"; import { project_id } from "@cocalc/project/data"; +import { delay } from "awaiting"; const logger = getLogger("project:nats:connection"); let nc: Awaited> | null = null; export default async function getConnection() { if (nc == null || (nc as any).protocol?.isClosed?.()) { + nc = null; logger.debug("initializing nats cocalc project connection"); if (!process.env.COCALC_NATS_JWT) { throw Error("environment variable COCALC_NATS_JWT *must* be set"); } const inboxPrefix = getInboxPrefix({ project_id }); logger.debug("Using ", { inboxPrefix }); - nc = await connect({ - ...CONNECT_OPTIONS, - authenticator: jwtAuthenticator(process.env.COCALC_NATS_JWT), - inboxPrefix, - }); - logger.debug(`connected to ${nc.getServer()}`); + let d = 3000; + while (nc == null) { + try { + nc = await connect({ + ...CONNECT_OPTIONS, + authenticator: jwtAuthenticator(process.env.COCALC_NATS_JWT), + inboxPrefix, + }); + logger.debug(`connected to ${nc.getServer()}`); + } catch (err) { + d = Math.min(15000, d * 1.2) + Math.random(); + logger.debug(`ERROR connecting; will retry in ${d}ms. err=${err}`); + await delay(d); + } + } } return nc!; } From 9f95980dcce54bd56773182abec3bfc51e4963a8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 19 Feb 2025 15:36:30 +0000 Subject: [PATCH 237/281] nats: fix project auth --- src/packages/project/nats/env.ts | 6 ++++++ src/packages/project/nats/open-files.ts | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/packages/project/nats/env.ts b/src/packages/project/nats/env.ts index 75eb7bceb9..b9f4866a9d 100644 --- a/src/packages/project/nats/env.ts +++ b/src/packages/project/nats/env.ts @@ -1,9 +1,15 @@ import { sha1 } from "@cocalc/backend/sha1"; import getConnection from "./connection"; import { JSONCodec } from "nats"; +import { setNatsClient } from "@cocalc/nats/client"; const jc = JSONCodec(); export async function getEnv() { const nc = await getConnection(); return { sha1, nc, jc }; } + +export function init() { + setNatsClient({ getNatsEnv: getEnv }); +} +init(); diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index b10fecf93d..781859d291 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -71,7 +71,8 @@ import { } from "@cocalc/nats/compute/manager"; // ensure nats connection stuff is initialized -import "@cocalc/backend/nats"; +import "@cocalc/project/nats/env"; +import { chdir } from "node:process"; const logger = getLogger("project:nats:open-files"); @@ -85,6 +86,10 @@ let computeServers: ComputeServerManager | null = null; export async function init() { logger.debug("init"); + if (process.env.HOME) { + chdir(process.env.HOME); + } + openFiles = await createOpenFiles(); computeServers = computeServerManager({ project_id }); From 4738cb912c7b37a894f32ac8126413ecf755f45e Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 19 Feb 2025 18:04:27 +0000 Subject: [PATCH 238/281] nats: implement time sync using nats jetstream --- src/packages/frontend/nats/client.ts | 13 +++ src/packages/nats/client.ts | 11 +++ src/packages/nats/service/service.ts | 5 ++ src/packages/nats/sync/kv.ts | 2 +- src/packages/nats/time.ts | 120 +++++++++++++++++++++++++++ 5 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 src/packages/nats/time.ts diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 5f012f174a..1b5cc126cf 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -46,6 +46,7 @@ import { computeServerManager, type Options as ComputeServerManagerOptions, } from "@cocalc/nats/compute/manager"; +import time, { getTime, getSkew as getClockSkew } from "@cocalc/nats/time"; export class NatsClient { client: WebappClient; @@ -476,6 +477,18 @@ export class NatsClient { await M.init(); return M; }; + + time = () => { + return time(); + }; + + getTime = async () => { + return await getTime(); + }; + + getClockSkew = async () => { + return await getClockSkew(); + }; } function setDeleted({ project_id, path, deleted }) { diff --git a/src/packages/nats/client.ts b/src/packages/nats/client.ts index 4ae5e91fc7..98b2329388 100644 --- a/src/packages/nats/client.ts +++ b/src/packages/nats/client.ts @@ -1,12 +1,16 @@ import type { NatsEnv, NatsEnvFunction } from "@cocalc/nats/types"; +import { init } from "./time"; interface Client { getNatsEnv: NatsEnvFunction; + account_id?: string; + project_id?: string; } let globalClient: null | Client = null; export function setNatsClient(client: Client) { globalClient = client; + setTimeout(init, 1); } export async function getEnv(): Promise { @@ -15,3 +19,10 @@ export async function getEnv(): Promise { } return await globalClient.getNatsEnv(); } + +export function getClient(): Client { + if (globalClient == null) { + throw Error("must set the global NATS client"); + } + return globalClient; +} diff --git a/src/packages/nats/service/service.ts b/src/packages/nats/service/service.ts index 947800bc40..679501eca4 100644 --- a/src/packages/nats/service/service.ts +++ b/src/packages/nats/service/service.ts @@ -38,6 +38,8 @@ export interface ServiceCall extends ServiceDescription { mesg: any; timeout?: number; env?: NatsEnv; + // if true, call returns the raw response message, with no decoding or error wrapping. + raw?: boolean; } export async function callNatsService(opts: ServiceCall): Promise { @@ -51,6 +53,9 @@ export async function callNatsService(opts: ServiceCall): Promise { resp = await nc.request(subject, jc.encode(opts.mesg), { timeout, }); + if (opts.raw) { + return resp; + } } catch (err) { if (err.name == "NatsError") { const p = opts.path ? `${trunc_middle(opts.path, 64)}:` : ""; diff --git a/src/packages/nats/sync/kv.ts b/src/packages/nats/sync/kv.ts index 83f891ab5e..97f5ed6955 100644 --- a/src/packages/nats/sync/kv.ts +++ b/src/packages/nats/sync/kv.ts @@ -64,7 +64,7 @@ export class KV extends EventEmitter { }, set(target, prop, value) { prop = String(prop); - if (prop == "_eventsCount" || prop == "_events") { + if (prop == "_eventsCount" || prop == "_events" || prop == "close") { target[prop] = value; return true; } diff --git a/src/packages/nats/time.ts b/src/packages/nats/time.ts new file mode 100644 index 0000000000..bc07da4338 --- /dev/null +++ b/src/packages/nats/time.ts @@ -0,0 +1,120 @@ +/* +Time sync entirely using nats itself. + +To use this, call the default export, which is a sync +function that returns the current sync'd time, or +throws an error if the first time sync hasn't succeeded. +This gets initialized by default on load of your process. + +It works using a key:value store via jetstream, +which is complicated overall. Normal request/reply +messages don't seem to have a timestamp, so I couldn't +use them. + +import time, {getTime} from "@cocalc/nats/time"; + +// sync - throws if hasn't connected and sync'd the first time: +time(); + +// async -- will wait to connect and tries to sync if haven't done so yet. Otherwise same as sync: +await getTime(); + +*/ + +import { dkv as createDkv } from "@cocalc/nats/sync/dkv"; +import { getClient, getEnv } from "@cocalc/nats/client"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { randomId } from "@cocalc/nats/names"; +import { callback, delay } from "awaiting"; +import { nanos } from "@cocalc/nats/util"; + +// max time to try when syncing +const TIMEOUT = 3 * 1000; + +// sync clock this frequently once it has sync'd once +const INTERVAL_GOOD = 15 * 1000 * 60; +const INTERVAL_BAD = 15 * 1000; + +export function init() { + syncLoop(); +} + +let state = "running"; +export function close() { + state = "closed"; +} + +async function syncLoop() { + while (state != "closed") { + try { + await getSkew(); + if (state == "closed") return; + await delay(INTERVAL_GOOD); + } catch (err) { + console.log("WARNING: failed to sync clock ", err); + if (state == "closed") return; + await delay(INTERVAL_BAD); + } + } +} + +let dkv: any = null; +const initDkv = reuseInFlight(async () => { + const { account_id, project_id } = getClient(); + dkv = await createDkv({ + account_id, + project_id, + env: await getEnv(), + name: "time", + limits: { + max_age: nanos(4 * TIMEOUT), + }, + }); +}); + +// skew = amount to add to our clock to get sync'd clock +export let skew: number | null = null; + +export async function getSkew(): Promise { + if (dkv == null) { + await initDkv(); + } + const start = Date.now(); + const id = randomId(); + dkv.set(id, ""); + const f = (cb) => { + const handle = ({ key }) => { + const end = Date.now(); + if (key == id) { + clearTimeout(timer); + dkv.removeListener("change", handle); + const serverTime = dkv.time(key)?.valueOf(); + dkv.delete(key); + const rtt = end - start; + skew = serverTime - (start + rtt / 2); + cb(undefined, skew); + } + }; + dkv.on("change", handle); + let timer = setTimeout(() => { + dkv.removeListener("change", handle); + dkv.delete(id); + cb("timeout"); + }, TIMEOUT); + }; + return await callback(f); +} + +export async function getTime(): Promise { + if (skew == null) { + await getSkew(); + } + return time(); +} + +export default function time(): Date { + if (skew == null) { + throw Error("clock skew not known"); + } + return new Date(Date.now() + skew); +} From ee4d6570c74fc0324e43d922e5dd2b56dbdb230d Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 19 Feb 2025 19:40:02 +0000 Subject: [PATCH 239/281] nats time and open files: use sync'd time explicitly --- src/packages/nats/client.ts | 1 + src/packages/nats/sync/open-files.ts | 47 ++++++++----------------- src/packages/nats/time.ts | 1 + src/packages/project/client.ts | 10 +++--- src/packages/project/nats/env.ts | 3 +- src/packages/project/nats/open-files.ts | 16 +++------ 6 files changed, 28 insertions(+), 50 deletions(-) diff --git a/src/packages/nats/client.ts b/src/packages/nats/client.ts index 98b2329388..e2093ee5b4 100644 --- a/src/packages/nats/client.ts +++ b/src/packages/nats/client.ts @@ -5,6 +5,7 @@ interface Client { getNatsEnv: NatsEnvFunction; account_id?: string; project_id?: string; + compute_server_id?: number; } let globalClient: null | Client = null; diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 3b99d6c503..75068fdbae 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -33,6 +33,7 @@ import { type NatsEnv, type State } from "@cocalc/nats/types"; import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; import { nanos } from "@cocalc/nats/util"; import { EventEmitter } from "events"; +import time from "@cocalc/nats/time"; // info about interest in open files (and also what was explicitly deleted) older // than this is automatically purged. @@ -41,14 +42,8 @@ const MAX_AGE_MS = 7 * (1000 * 60 * 60 * 24); export interface Entry { // path to file relative to HOME path: string; - // if true, then file should be opened, managed, and watched - // by home base or compute server - open?: boolean; - // last time this entry was changed -- this is automatically set - // correctly by the NATS server in a consistent way: - // https://github.com/nats-io/nats-server/discussions/3095 - time?: Date; - count?: number; + // a web browser has the file open at this point in time (in ms) + time?: number; // if the file was removed from disk (and not immmediately written back), // then deleted gets set to the time when this happened (in ms since epoch) // and the file is closed on the backend. It won't be re-opened until @@ -148,12 +143,8 @@ export class OpenFiles extends EventEmitter { return dkv; }; - private set = (key, entry: Entry) => { - // remove time field, if there -- it is filled in automatically - entry = { ...entry }; - // @ts-ignore - delete entry.time; - this.getDkv().set(key, entry); + private set = (path, entry: Entry) => { + this.getDkv().set(path, entry); }; // When a client has a file open, they should periodically @@ -167,10 +158,16 @@ export class OpenFiles extends EventEmitter { // n = sequence number to make sure a write happens, which updates // server assigned timestamp. const cur = dkv.get(path); + let t; + try { + t = time().valueOf(); + } catch { + // give up -- try again once initialized + return; + } this.set(path, { ...cur, - open: true, - count: (cur?.count ?? 0) + 1, + time: t, }); }; @@ -187,16 +184,6 @@ export class OpenFiles extends EventEmitter { } }; - // causes file to be immediately closed on backend - // no matter what, unrelated to how many users have it - // open or what type of file it is. Obviously, frontend - // clients also may need to pay attention to this, since they - // can just immediately reopen the file. - closeFile = (path: string) => { - const dkv = this.getDkv(); - this.set(path, { ...dkv.get(path), open: false }); - }; - setDeleted = (path: string) => { const dkv = this.getDkv(); this.set(path, { ...dkv.get(path), deleted: Date.now() }); @@ -220,7 +207,7 @@ export class OpenFiles extends EventEmitter { getAll = (): Entry[] => { const x = this.getDkv().getAll(); return Object.keys(x).map((path) => { - return { ...x[path], path, time: this.time(path) }; + return { ...x[path], path }; }); }; @@ -229,7 +216,7 @@ export class OpenFiles extends EventEmitter { if (x == null) { return x; } - return { ...x, path, time: this.time(path) }; + return { ...x, path }; }; delete = (path) => { @@ -247,8 +234,4 @@ export class OpenFiles extends EventEmitter { hasUnsavedChanges = () => { return this.getDkv().hasUnsavedChanges(); }; - - time = (path?: string) => { - return this.getDkv().time(path); - }; } diff --git a/src/packages/nats/time.ts b/src/packages/nats/time.ts index bc07da4338..8d94fb3e03 100644 --- a/src/packages/nats/time.ts +++ b/src/packages/nats/time.ts @@ -61,6 +61,7 @@ async function syncLoop() { let dkv: any = null; const initDkv = reuseInFlight(async () => { const { account_id, project_id } = getClient(); + console.log({ account_id, project_id, client: getClient() }); dkv = await createDkv({ account_id, project_id, diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index 31a5012c1f..2ca26a0796 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -59,7 +59,6 @@ import { type CreateNatsServiceFunction, } from "@cocalc/nats/service"; import type { NatsEnvFunction } from "@cocalc/nats/types"; -import { setNatsClient } from "@cocalc/nats/client"; const winston = getLogger("client"); @@ -86,21 +85,22 @@ if (!DEBUG) { winston.info(`create ${DEBUG_FILE} for much more verbose logging`); } -let client: Client; +let client: Client | null = null; export function init() { if (client != null) { return client; } client = new Client(); - setNatsClient(client); return client; } export function getClient(): Client { if (client == null) { init(); - //throw Error("BUG: Client not initialized!"); + } + if (client == null) { + throw Error("BUG: Client not initialized!"); } return client; } @@ -110,7 +110,7 @@ let ALREADY_CREATED = false; type HubCB = CB; export class Client extends EventEmitter implements ProjectClientInterface { - private project_id: string; + public readonly project_id: string; private _connected: boolean; private _hub_callbacks: { diff --git a/src/packages/project/nats/env.ts b/src/packages/project/nats/env.ts index b9f4866a9d..61980e7c18 100644 --- a/src/packages/project/nats/env.ts +++ b/src/packages/project/nats/env.ts @@ -2,6 +2,7 @@ import { sha1 } from "@cocalc/backend/sha1"; import getConnection from "./connection"; import { JSONCodec } from "nats"; import { setNatsClient } from "@cocalc/nats/client"; +import { compute_server_id, project_id } from "@cocalc/project/data"; const jc = JSONCodec(); export async function getEnv() { @@ -10,6 +11,6 @@ export async function getEnv() { } export function init() { - setNatsClient({ getNatsEnv: getEnv }); + setNatsClient({ getNatsEnv: getEnv, project_id, compute_server_id }); } init(); diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index 781859d291..eca2182325 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -144,8 +144,8 @@ export function terminate() { computeServers = null; } -function getCutoff() { - return new Date(Date.now() - 2.5 * NATS_OPEN_FILE_TOUCH_INTERVAL); +function getCutoff(): number { + return Date.now() - 2.5 * NATS_OPEN_FILE_TOUCH_INTERVAL; } function computeServerId(path: string): number { @@ -154,7 +154,6 @@ function computeServerId(path: string): number { async function handleChange({ path, - open, time, deleted, id, @@ -162,7 +161,7 @@ async function handleChange({ if (id == null) { id = computeServerId(path); } - logger.debug("handleChange", { path, open, time, deleted, id }); + logger.debug("handleChange", { path, time, deleted, id }); const syncDoc = openDocs[path]; const isOpenHere = syncDoc != null; @@ -186,14 +185,7 @@ async function handleChange({ } } - if (!open) { - if (isOpenHere) { - logger.debug("handleChange: closing", { path }); - closeDoc(path); - } - return; - } - if (time != null && open && time >= getCutoff()) { + if (time != null && time >= getCutoff()) { if (!isOpenHere) { logger.debug("handleChange: opening", { path }); // users actively care about this file being opened HERE, but it isn't From b5e6bc5c71cea6d3d0bdfd2ba78f0c26c4802940 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 19 Feb 2025 20:13:31 +0000 Subject: [PATCH 240/281] nats: change delete in open-files so can properly resolve merge conflicts --- src/packages/frontend/nats/client.ts | 26 +++---- src/packages/nats/sync/open-files.ts | 91 ++++++++++++++++++------- src/packages/nats/time.ts | 27 +++----- src/packages/project/nats/open-files.ts | 4 +- 4 files changed, 92 insertions(+), 56 deletions(-) diff --git a/src/packages/frontend/nats/client.ts b/src/packages/frontend/nats/client.ts index 1b5cc126cf..204d78a079 100644 --- a/src/packages/frontend/nats/client.ts +++ b/src/packages/frontend/nats/client.ts @@ -46,7 +46,7 @@ import { computeServerManager, type Options as ComputeServerManagerOptions, } from "@cocalc/nats/compute/manager"; -import time, { getTime, getSkew as getClockSkew } from "@cocalc/nats/time"; +import getTime, { getSkew } from "@cocalc/nats/time"; export class NatsClient { client: WebappClient; @@ -353,16 +353,20 @@ export class NatsClient { delete this.openFilesCache[project_id]; }); openFiles.on("change", (entry) => { - if (entry.deleted) { - setDeleted({ project_id, path: entry.path, deleted: entry.deleted }); + if (entry.deleted?.deleted) { + setDeleted({ + project_id, + path: entry.path, + deleted: entry.deleted.time, + }); } else { setNotDeleted({ project_id, path: entry.path }); } }); const recentlyDeletedPaths: any = {}; for (const { path, deleted } of openFiles.getAll()) { - if (deleted) { - recentlyDeletedPaths[path] = deleted; + if (deleted?.deleted) { + recentlyDeletedPaths[path] = deleted.time; } } const store = redux.getProjectStore(project_id); @@ -478,16 +482,12 @@ export class NatsClient { return M; }; - time = () => { - return time(); + getTime = (): number => { + return getTime(); }; - getTime = async () => { - return await getTime(); - }; - - getClockSkew = async () => { - return await getClockSkew(); + getSkew = async (): Promise => { + return await getSkew(); }; } diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index 75068fdbae..a3cfd2e8ee 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -33,15 +33,22 @@ import { type NatsEnv, type State } from "@cocalc/nats/types"; import { dkv, type DKV } from "@cocalc/nats/sync/dkv"; import { nanos } from "@cocalc/nats/util"; import { EventEmitter } from "events"; -import time from "@cocalc/nats/time"; +import getTime from "@cocalc/nats/time"; // info about interest in open files (and also what was explicitly deleted) older // than this is automatically purged. const MAX_AGE_MS = 7 * (1000 * 60 * 60 * 24); -export interface Entry { - // path to file relative to HOME - path: string; +interface Deleted { + // what deleted state is + deleted: boolean; + // when deleted state set + time: number; +} + +// IMPORTANT: if you add/change any fields below, be sure to update +// the merge conflict function! +export interface KVEntry { // a web browser has the file open at this point in time (in ms) time?: number; // if the file was removed from disk (and not immmediately written back), @@ -50,7 +57,46 @@ export interface Entry { // either (1) the file is created on disk again, or (2) deleted is cleared. // Note: the actual time here isn't really important -- what matter is the number // is nonzero. It's just used for a display to the user. - deleted?: number; + // We store the deleted state *and* when this was set, so that in case + // of merge conflict we can do something sensible. + deleted?: Deleted; +} + +function resolveMergeConflict(local: KVEntry, remote: KVEntry): KVEntry { + const time = mergeTime(remote?.time, local?.time); + const deleted = mergeDeleted(remote?.deleted, local?.deleted); + return { + time, + deleted, + }; +} + +export interface Entry extends KVEntry { + // path to file relative to HOME + path: string; +} + +function mergeTime( + a: number | undefined, + b: number | undefined, +): number | undefined { + // time of interest should clearly always be the largest known value so far. + if (a == null && b == null) { + return undefined; + } + return Math.max(a ?? 0, b ?? 0); +} + +function mergeDeleted(a: Deleted | undefined, b: Deleted | undefined) { + if (a == null) { + return b; + } + if (b == null) { + return a; + } + // now both a and b are not null, so some merge is needed: we + // use last write wins. + return a.time >= b.time ? a : b; } interface Options { @@ -94,7 +140,7 @@ export class OpenFiles extends EventEmitter { }; init = async () => { - const d = await dkv({ + const d = await dkv({ name: "open-files", project_id: this.project_id, env: this.env, @@ -103,12 +149,7 @@ export class OpenFiles extends EventEmitter { }, noAutosave: this.noAutosave, noCache: this.noCache, - merge: ({ local, remote }) => { - // resolve conflicts by merging object state. This is important so, e.g., the - // deleted state doesn't get overwritten on reconnect by clients that didn't know. - //console.log("merge", local, remote, { ...remote, ...local }); - return { ...remote, ...local }; - }, + merge: ({ local, remote }) => resolveMergeConflict(local, remote), }); this.dkv = d; d.on("change", ({ key: path }) => { @@ -143,7 +184,7 @@ export class OpenFiles extends EventEmitter { return dkv; }; - private set = (path, entry: Entry) => { + private set = (path, entry: KVEntry) => { this.getDkv().set(path, entry); }; @@ -158,16 +199,16 @@ export class OpenFiles extends EventEmitter { // n = sequence number to make sure a write happens, which updates // server assigned timestamp. const cur = dkv.get(path); - let t; + let time; try { - t = time().valueOf(); + time = getTime(); } catch { // give up -- try again once initialized return; } this.set(path, { ...cur, - time: t, + time, }); }; @@ -186,22 +227,22 @@ export class OpenFiles extends EventEmitter { setDeleted = (path: string) => { const dkv = this.getDkv(); - this.set(path, { ...dkv.get(path), deleted: Date.now() }); + this.set(path, { + ...dkv.get(path), + deleted: { deleted: true, time: getTime() }, + }); }; isDeleted = (path: string) => { - return !!this.getDkv().get(path)?.deleted; + return !!this.getDkv().get(path)?.deleted?.deleted; }; setNotDeleted = (path: string) => { const dkv = this.getDkv(); - let cur = dkv.get(path); - if (cur == null) { - return; - } - cur = { ...cur }; - delete cur.deleted; - this.set(path, cur); + this.set(path, { + ...dkv.get(path), + deleted: { deleted: false, time: getTime() }, + }); }; getAll = (): Entry[] => { diff --git a/src/packages/nats/time.ts b/src/packages/nats/time.ts index 8d94fb3e03..b781b6e3b7 100644 --- a/src/packages/nats/time.ts +++ b/src/packages/nats/time.ts @@ -2,7 +2,7 @@ Time sync entirely using nats itself. To use this, call the default export, which is a sync -function that returns the current sync'd time, or +function that returns the current sync'd time (in ms since epoch), or throws an error if the first time sync hasn't succeeded. This gets initialized by default on load of your process. @@ -11,13 +11,15 @@ which is complicated overall. Normal request/reply messages don't seem to have a timestamp, so I couldn't use them. -import time, {getTime} from "@cocalc/nats/time"; +import getTime, {getSkew} from "@cocalc/nats/time"; -// sync - throws if hasn't connected and sync'd the first time: -time(); +// sync - this throws if hasn't connected and sync'd the first time: + +getTime(); // -- ms since the epoch // async -- will wait to connect and tries to sync if haven't done so yet. Otherwise same as sync: -await getTime(); +// once this works you can definitely call getTime henceforth. +await getSkew(); */ @@ -73,7 +75,7 @@ const initDkv = reuseInFlight(async () => { }); }); -// skew = amount to add to our clock to get sync'd clock +// skew = amount in ms to subtract from our clock to get sync'd clock export let skew: number | null = null; export async function getSkew(): Promise { @@ -92,7 +94,7 @@ export async function getSkew(): Promise { const serverTime = dkv.time(key)?.valueOf(); dkv.delete(key); const rtt = end - start; - skew = serverTime - (start + rtt / 2); + skew = start + rtt / 2 - serverTime; cb(undefined, skew); } }; @@ -106,16 +108,9 @@ export async function getSkew(): Promise { return await callback(f); } -export async function getTime(): Promise { - if (skew == null) { - await getSkew(); - } - return time(); -} - -export default function time(): Date { +export default function getTime(): number { if (skew == null) { throw Error("clock skew not known"); } - return new Date(Date.now() + skew); + return Date.now() - skew; } diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index eca2182325..a5522318a1 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -173,7 +173,7 @@ async function handleChange({ return; } - if (deleted) { + if (deleted?.deleted) { if (await exists(path)) { // it's back openFiles?.setNotDeleted(path); @@ -255,7 +255,7 @@ async function checkForFileDeletion(path: string) { if (entry == null) { return; } - if (entry.deleted) { + if (entry.deleted?.deleted) { // already set as deleted -- shouldn't still be opened await closeDoc(entry.path); } else { From 03ca5bc6141c41ace4a0d51ac963ffb8a0207bbc Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 19 Feb 2025 21:09:19 +0000 Subject: [PATCH 241/281] nats open files: track exactly which backend has file opened and when --- src/packages/nats/sync/open-files.ts | 56 +++++++++ src/packages/project/nats/open-files.ts | 156 ++++++++++++++---------- 2 files changed, 147 insertions(+), 65 deletions(-) diff --git a/src/packages/nats/sync/open-files.ts b/src/packages/nats/sync/open-files.ts index a3cfd2e8ee..096636388a 100644 --- a/src/packages/nats/sync/open-files.ts +++ b/src/packages/nats/sync/open-files.ts @@ -46,6 +46,13 @@ interface Deleted { time: number; } +interface Backend { + // who has it opened -- the compute_server_id (0 for project) + id: number; + // when they last reported having it opened + time: number; +} + // IMPORTANT: if you add/change any fields below, be sure to update // the merge conflict function! export interface KVEntry { @@ -60,14 +67,22 @@ export interface KVEntry { // We store the deleted state *and* when this was set, so that in case // of merge conflict we can do something sensible. deleted?: Deleted; + + // if file is actively opened on a compute server, then it sets + // this entry. Right when it closes the file, it clears this. + // If it gets killed/broken and doesn't have a chance to clear it, then + // backend.time can be used to decide this isn't valid. + backend?: Backend; } function resolveMergeConflict(local: KVEntry, remote: KVEntry): KVEntry { const time = mergeTime(remote?.time, local?.time); const deleted = mergeDeleted(remote?.deleted, local?.deleted); + const backend = mergeBackend(remote?.backend, local?.backend); return { time, deleted, + backend, }; } @@ -99,6 +114,20 @@ function mergeDeleted(a: Deleted | undefined, b: Deleted | undefined) { return a.time >= b.time ? a : b; } +function mergeBackend(a: Backend | undefined, b: Backend | undefined) { + if (a == null) { + return b; + } + if (b == null) { + return a; + } + // now both a and b are not null, so some merge is needed: we + // use last write wins. + // NOTE: This should likely not happen or only happen for a moment and + // would be worrisome, but quickly sort itself out. + return a.time >= b.time ? a : b; +} + interface Options { env: NatsEnv; project_id: string; @@ -245,6 +274,33 @@ export class OpenFiles extends EventEmitter { }); }; + // set that id is the backend with the file open. + // This should be called by that backend periodically + // when it has the file opened. + setBackend = (path: string, id: number) => { + const dkv = this.getDkv(); + this.set(path, { + ...dkv.get(path), + backend: { id, time: getTime() }, + }); + }; + + // get current backend that has file opened. + getBackend = (path: string): Backend | undefined => { + return this.getDkv().get(path)?.backend; + }; + + // ONLY if backend for path is currently set to id, then clear + // the backend field. + setNotBackend = (path: string, id: number) => { + const dkv = this.getDkv(); + const cur = { ...dkv.get(path) }; + if (cur?.backend?.id == id) { + delete cur.backend; + this.set(path, cur); + } + }; + getAll = (): Entry[] => { const x = this.getDkv().getAll(); return Object.keys(x).map((path) => { diff --git a/src/packages/project/nats/open-files.ts b/src/packages/project/nats/open-files.ts index a5522318a1..1fc71b110b 100644 --- a/src/packages/project/nats/open-files.ts +++ b/src/packages/project/nats/open-files.ts @@ -100,7 +100,7 @@ export async function init() { } const entry = openFiles?.get(path); if (entry != null) { - await handleChange({ ...entry, path, id }); + await handleChange({ ...entry, id }); } else { await closeDoc(path); } @@ -114,6 +114,8 @@ export async function init() { // start loop to watch for and close files that aren't touched frequently: closeIgnoredFilesLoop(); + // periodically update timestamp on backend for files we have open + touchOpenFilesLoop(); // watch if any file that is currently opened on this host gets deleted, // and if so, mark it as such, and set it to closed. watchForFileDeletionLoop(); @@ -156,16 +158,21 @@ async function handleChange({ path, time, deleted, + backend, id, }: OpenFileEntry & { id?: number }) { if (id == null) { id = computeServerId(path); } - logger.debug("handleChange", { path, time, deleted, id }); + logger.debug("handleChange", { path, time, deleted, backend, id }); const syncDoc = openDocs[path]; const isOpenHere = syncDoc != null; if (id != compute_server_id) { + if (backend?.id == compute_server_id) { + // we are definitely not the backend right now. + openFiles?.setNotBackend(path, compute_server_id); + } // only thing we should do is close it if it is open. if (isOpenHere) { await closeDoc(path); @@ -179,7 +186,7 @@ async function handleChange({ openFiles?.setNotDeleted(path); } else { if (isOpenHere) { - closeDoc(path); + await closeDoc(path); } return; } @@ -189,7 +196,7 @@ async function handleChange({ if (!isOpenHere) { logger.debug("handleChange: opening", { path }); // users actively care about this file being opened HERE, but it isn't - openDoc(path); + await openDoc(path); } return; } @@ -205,7 +212,7 @@ function supportAutoclose(path: string): boolean { } async function closeIgnoredFilesLoop() { - while (openFiles != null && openFiles.state == "connected") { + while (openFiles?.state == "connected") { await delay(NATS_OPEN_FILE_TOUCH_INTERVAL); if (openFiles?.state != "connected") { return; @@ -225,6 +232,7 @@ async function closeIgnoredFilesLoop() { if ( entry != null && entry.time != null && + openDocs[entry.path] != null && entry.time <= cutoff && supportAutoclose(entry.path) ) { @@ -235,6 +243,15 @@ async function closeIgnoredFilesLoop() { } } +async function touchOpenFilesLoop() { + while (openFiles?.state == "connected" && openDocs != null) { + for (const path in openDocs) { + openFiles.setBackend(path, compute_server_id); + } + await delay(NATS_OPEN_FILE_TOUCH_INTERVAL); + } +} + async function checkForFileDeletion(path: string) { if (openFiles == null) { return; @@ -314,80 +331,89 @@ async function watchForFileDeletionLoop() { const closeDoc = reuseInFlight(async (path: string) => { logger.debug("close", { path }); - const doc = openDocs[path]; - if (doc == null) { - return; - } - delete openDocs[path]; try { - await doc.close(); - } catch (err) { - logger.debug(`WARNING -- issue closing doc -- ${err}`); - openFiles?.setError(path, err); + const doc = openDocs[path]; + if (doc == null) { + return; + } + delete openDocs[path]; + try { + await doc.close(); + } catch (err) { + logger.debug(`WARNING -- issue closing doc -- ${err}`); + openFiles?.setError(path, err); + } + } finally { + if (openDocs[path] == null) { + openFiles?.setNotBackend(path, compute_server_id); + } } }); const openDoc = reuseInFlight(async (path: string) => { // todo -- will be async and needs to handle SyncDB and all the config... logger.debug("openDoc", { path }); + try { + const doc = openDocs[path]; + if (doc != null) { + return; + } - const doc = openDocs[path]; - if (doc != null) { - return; - } - - if (path.endsWith(".term")) { - const service = await createTerminalService(path); - openDocs[path] = service; - return; - } - - const client = getClient(); - const doctype = await getSyncDocType({ - project_id, - path, - client, - }); - logger.debug("openDoc got", { path, doctype }); + if (path.endsWith(".term")) { + const service = await createTerminalService(path); + openDocs[path] = service; + return; + } - let syncdoc; - if (doctype.type == "string") { - syncdoc = new SyncString({ - ...doctype.opts, + const client = getClient(); + const doctype = await getSyncDocType({ project_id, path, client, }); - } else { - syncdoc = new SyncDB({ - ...doctype.opts, - project_id, - path, - client, - }); - } - openDocs[path] = syncdoc; + logger.debug("openDoc got", { path, doctype }); + + let syncdoc; + if (doctype.type == "string") { + syncdoc = new SyncString({ + ...doctype.opts, + project_id, + path, + client, + }); + } else { + syncdoc = new SyncDB({ + ...doctype.opts, + project_id, + path, + client, + }); + } + openDocs[path] = syncdoc; - syncdoc.on("error", (err) => { - closeDoc(path); - openFiles?.setError(path, err); - logger.debug(`syncdoc error -- ${err}`, path); - }); + syncdoc.on("error", (err) => { + closeDoc(path); + openFiles?.setError(path, err); + logger.debug(`syncdoc error -- ${err}`, path); + }); - // Extra backend support in some cases, e.g., Jupyter, Sage, etc. - const ext = filename_extension(path); - switch (ext) { - case "sage-jupyter2": - logger.debug("initializing Jupyter backend for ", path); - await get_blob_store(); // make sure jupyter blobstore is available - await initJupyterRedux(syncdoc, client); - const path1 = original_path(syncdoc.get_path()); - syncdoc.on("closed", async () => { - logger.debug("removing Jupyter backend for ", path1); - await removeJupyterRedux(path1, project_id); - }); - break; + // Extra backend support in some cases, e.g., Jupyter, Sage, etc. + const ext = filename_extension(path); + switch (ext) { + case "sage-jupyter2": + logger.debug("initializing Jupyter backend for ", path); + await get_blob_store(); // make sure jupyter blobstore is available + await initJupyterRedux(syncdoc, client); + const path1 = original_path(syncdoc.get_path()); + syncdoc.on("closed", async () => { + logger.debug("removing Jupyter backend for ", path1); + await removeJupyterRedux(path1, project_id); + }); + break; + } + } finally { + if (openDocs[path] != null) { + openFiles?.setBackend(path, compute_server_id); + } } - - return; }); From 4bbd937d4b47f2ec8ba8cd706f7d5419d49830c5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 19 Feb 2025 23:45:41 +0000 Subject: [PATCH 242/281] nats: start wokr on streaming file get service --- src/packages/nats/files/get.ts | 69 ++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/packages/nats/files/get.ts diff --git a/src/packages/nats/files/get.ts b/src/packages/nats/files/get.ts new file mode 100644 index 0000000000..b77af33648 --- /dev/null +++ b/src/packages/nats/files/get.ts @@ -0,0 +1,69 @@ +/* +A NATS service that uses requestMany, takes as input a filename apth, and streams all +the binary data from that path. + +We use headers to add sequence numbers into the response messages. + +This is useful to implement: + +- an http server for downloading any file, even large ones. + + +IDEAS: + +- we could also implement a version of this that takes a directory +as input, runs compressed tar on it, and pipes the output into +response messages. We could then implement streaming download of +a tarball of a directory tree, or also copying a directory tree from +one place to another (without using rsync). I've done this already +over a websocket for compute servers, so would just copy that code. +*/ + +import { getEnv } from "@cocalc/nats/client"; +import { projectSubject } from "@cocalc/nats/names"; + +function getSubject({ project_id, compute_server_id }) { + return projectSubject({ + project_id, + compute_server_id, + service: "files-get", + }); +} + +export async function createServer({ + readFromDisk, + project_id, + compute_server_id, +}) { + const { nc, jc } = await getEnv(); + const subject = getSubject({ + project_id, + compute_server_id, + }); + console.log(subject); + const sub = nc.subscribe(subject); + for await (const mesg of sub) { + handleMessage(mesg); + } +} + +async function handleMessage(mesg) { + mesg.respond("xxx"); + mesg.respond("yyy"); + mesg.respond(); +} + +export async function getFile({ project_id, compute_server_id, path }) { + const { nc, jc } = await getEnv(); + const subject = getSubject({ + project_id, + compute_server_id, + }); + const v: any = []; + for await (const resp of await nc.requestMany(subject, jc.encode({ path }))) { + console.log(resp); + v.push(resp); + } + console.log("done"); + return v; +} From 2f038d0544998ae82567c251aa9b6fa79ff39568 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 20 Feb 2025 01:01:04 +0000 Subject: [PATCH 243/281] nats: implement async generator to read file from compute server or project --- src/packages/nats/files/get.ts | 64 +++++++++++++++++++++++++++------- src/packages/nats/package.json | 13 +++++-- src/packages/pnpm-lock.yaml | 11 ++++++ 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/src/packages/nats/files/get.ts b/src/packages/nats/files/get.ts index b77af33648..b272f30f30 100644 --- a/src/packages/nats/files/get.ts +++ b/src/packages/nats/files/get.ts @@ -1,5 +1,8 @@ /* -A NATS service that uses requestMany, takes as input a filename apth, and streams all +Read a file from a project/compute server via an async generator, so it is memory +efficient. + +This is a NATS service that uses requestMany, takes as input a filename path, and streams all the binary data from that path. We use headers to add sequence numbers into the response messages. @@ -17,10 +20,20 @@ response messages. We could then implement streaming download of a tarball of a directory tree, or also copying a directory tree from one place to another (without using rsync). I've done this already over a websocket for compute servers, so would just copy that code. + + +DEVELOPMENT: + +~/cocalc/src/packages/backend$ node + +require('@cocalc/backend/nats'); a = require('@cocalc/nats/files/get'); a.createServer({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,createReadStream:require('fs').createReadStream}) + +for await (const chunk of await a.readFile({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,path:'/tmp/a.py'})) { console.log({chunk}); } */ import { getEnv } from "@cocalc/nats/client"; import { projectSubject } from "@cocalc/nats/names"; +import { Empty, headers } from "@nats-io/nats-core"; function getSubject({ project_id, compute_server_id }) { return projectSubject({ @@ -31,11 +44,11 @@ function getSubject({ project_id, compute_server_id }) { } export async function createServer({ - readFromDisk, + createReadStream, project_id, compute_server_id, }) { - const { nc, jc } = await getEnv(); + const { nc } = await getEnv(); const subject = getSubject({ project_id, compute_server_id, @@ -43,27 +56,54 @@ export async function createServer({ console.log(subject); const sub = nc.subscribe(subject); for await (const mesg of sub) { - handleMessage(mesg); + try { + await handleMessage(mesg, createReadStream); + const h = headers(); + h.append("done", ""); + mesg.respond(Empty, { headers: h }); + } catch (err) { + const h = headers(); + h.append("error", `${err}`); + mesg.respond(Empty, { headers: h }); + } } } -async function handleMessage(mesg) { - mesg.respond("xxx"); - mesg.respond("yyy"); - mesg.respond(); +async function handleMessage(mesg, createReadStream) { + const { jc } = await getEnv(); + const { path } = jc.decode(mesg.data); + let seq = 0; + for await (const chunk of createReadStream(path)) { + const h = headers(); + seq += 1; + h.append("seq", `${seq}`); + mesg.respond(chunk, { headers: h }); + } } -export async function getFile({ project_id, compute_server_id, path }) { +export async function* readFile({ project_id, compute_server_id, path }) { const { nc, jc } = await getEnv(); const subject = getSubject({ project_id, compute_server_id, }); const v: any = []; + let seq = 0; for await (const resp of await nc.requestMany(subject, jc.encode({ path }))) { - console.log(resp); - v.push(resp); + for (const [key, value] of resp.headers) { + if (key == "error") { + throw Error(value); + } else if (key == "done") { + return; + } else if (key == "seq") { + const next = parseInt(value); + if (next != seq + 1) { + throw Error("lost data"); + } + seq = next; + } + } + yield resp.data; } - console.log("done"); return v; } diff --git a/src/packages/nats/package.json b/src/packages/nats/package.json index 2028704591..3aacdc9dd5 100644 --- a/src/packages/nats/package.json +++ b/src/packages/nats/package.json @@ -18,9 +18,17 @@ "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "prepublishOnly": "pnpm test" }, - "files": ["dist/**", "README.md", "package.json"], + "files": [ + "dist/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["utilities", "nats", "cocalc"], + "keywords": [ + "utilities", + "nats", + "cocalc" + ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/comm": "workspace:*", @@ -28,6 +36,7 @@ "@cocalc/util": "workspace:*", "@nats-io/jetstream": "3.0.0-36", "@nats-io/kv": "3.0.0-30", + "@nats-io/nats-core": "3.0.0-51", "@nats-io/services": "3.0.0-25", "awaiting": "^3.0.0", "events": "3.3.0", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 7aaa016bd9..e28a4d8185 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1075,6 +1075,9 @@ importers: '@nats-io/kv': specifier: 3.0.0-30 version: 3.0.0-30 + '@nats-io/nats-core': + specifier: 3.0.0-51 + version: 3.0.0-51 '@nats-io/services': specifier: 3.0.0-25 version: 3.0.0-25 @@ -3966,6 +3969,9 @@ packages: '@nats-io/nats-core@3.0.0-50': resolution: {integrity: sha512-Kur1/yhzNrpcpu+OhsQ89k9Ge1woEWJd5FV3tpp0BtpRWMlIth3StBiADguynbKSQCkBAOUQ+C0kRnFW8zOIeg==} + '@nats-io/nats-core@3.0.0-51': + resolution: {integrity: sha512-BMNVBGPN1izxZ3zD17Z1XTz8OezIPzHK26a7ZCQFXt50eqXGrNrpnDNn8br5KrL2GCl5q3jnMh1jhUQHzIdZkA==} + '@nats-io/nkeys@2.0.2': resolution: {integrity: sha512-0JTyVl9P+UJyjUBDWP9589TuUKXJQ8tDkVRgi02X/MMzW997+4FykirvZEkIe6ZAhiLIBN+NpN8ULMMt6mDrbA==} engines: {node: '>=18.0.0'} @@ -14991,6 +14997,11 @@ snapshots: '@nats-io/nkeys': 2.0.2 '@nats-io/nuid': 2.0.3 + '@nats-io/nats-core@3.0.0-51': + dependencies: + '@nats-io/nkeys': 2.0.2 + '@nats-io/nuid': 2.0.3 + '@nats-io/nkeys@2.0.2': dependencies: tweetnacl: 1.0.3 From d3a8f84304cc413b1639d7c27f4f939aa50989f1 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 20 Feb 2025 02:55:50 +0000 Subject: [PATCH 244/281] nats: raw files -- work in progress --- src/packages/hub/package.json | 1 + src/packages/hub/proxy/handle-request.ts | 26 ++++++++++++++++++++++++ src/packages/hub/tsconfig.json | 3 ++- src/packages/nats/files/get.ts | 9 +++++++- src/packages/pnpm-lock.yaml | 3 +++ 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/packages/hub/package.json b/src/packages/hub/package.json index 41ea899fdc..311e63f1cb 100644 --- a/src/packages/hub/package.json +++ b/src/packages/hub/package.json @@ -13,6 +13,7 @@ "@cocalc/database": "workspace:*", "@cocalc/frontend": "workspace:*", "@cocalc/hub": "workspace:*", + "@cocalc/nats": "workspace:*", "@cocalc/next": "workspace:*", "@cocalc/server": "workspace:*", "@cocalc/static": "workspace:*", diff --git a/src/packages/hub/proxy/handle-request.ts b/src/packages/hub/proxy/handle-request.ts index 156541da03..09854f9302 100644 --- a/src/packages/hub/proxy/handle-request.ts +++ b/src/packages/hub/proxy/handle-request.ts @@ -9,6 +9,8 @@ import getLogger from "../logger"; import { stripBasePath } from "./util"; import { ProjectControlFunction } from "@cocalc/server/projects/control"; import siteUrl from "@cocalc/database/settings/site-url"; +import { parseReq } from "./parse"; +import { readFile } from "@cocalc/nats/files/get"; const logger = getLogger("proxy:handle-request"); @@ -76,6 +78,30 @@ export default function init({ projectControl, isPersonal }: Options) { } const url = stripBasePath(req.url); + // TODO: parseReq is called again in getTarget so need to refactor... + const { type, project_id } = parseReq(url, remember_me, api_key); + if (type == "raw") { + dbg("handling the request via NATS!"); + // TODO: do better, obviously + const i = url.indexOf("raw/"); + const compute_server_id = 0; + const path = url.slice(i + 4); + // worry decodeURI + dbg("NATs: get", { project_id, path, compute_server_id }); + const fileName = path; // todo + res.setHeader("Content-disposition", "attachment; filename=" + fileName); + res.setHeader("Content-type", "text/plain"); // todo + for await (const chunk of await readFile({ + project_id, + compute_server_id, + path, + })) { + res.write(chunk); + } + res.end(); + return; + } + const { host, port, internal_url } = await getTarget({ remember_me, api_key, diff --git a/src/packages/hub/tsconfig.json b/src/packages/hub/tsconfig.json index e38cd9b917..4c91795874 100644 --- a/src/packages/hub/tsconfig.json +++ b/src/packages/hub/tsconfig.json @@ -13,6 +13,7 @@ { "path": "../next/tsconfig-dist.json" }, { "path": "../server" }, { "path": "../static" }, - { "path": "../util" } + { "path": "../util" }, + { "path": "../nats" } ] } diff --git a/src/packages/nats/files/get.ts b/src/packages/nats/files/get.ts index b272f30f30..a9374ab89b 100644 --- a/src/packages/nats/files/get.ts +++ b/src/packages/nats/files/get.ts @@ -29,6 +29,11 @@ DEVELOPMENT: require('@cocalc/backend/nats'); a = require('@cocalc/nats/files/get'); a.createServer({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,createReadStream:require('fs').createReadStream}) for await (const chunk of await a.readFile({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,path:'/tmp/a.py'})) { console.log({chunk}); } + + +for await (const chunk of await a.readFile({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,path:'/projects/6b851643-360e-435e-b87e-f9a6ab64a8b1/cocalc/.git/objects/pack/pack-771f7fe4ee855601463be070cf9fb9afc91f84ac.pack'})) { console.log({chunk}); } + + */ import { getEnv } from "@cocalc/nats/client"; @@ -73,7 +78,9 @@ async function handleMessage(mesg, createReadStream) { const { jc } = await getEnv(); const { path } = jc.decode(mesg.data); let seq = 0; - for await (const chunk of createReadStream(path)) { + for await (const chunk of createReadStream(path, { + highWaterMark: 16384 * 16 * 2, + })) { const h = headers(); seq += 1; h.append("seq", `${seq}`); diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index e28a4d8185..a5fdd78e69 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -755,6 +755,9 @@ importers: '@cocalc/hub': specifier: workspace:* version: 'link:' + '@cocalc/nats': + specifier: workspace:* + version: link:../nats '@cocalc/next': specifier: workspace:* version: link:../next From 352961a4c31b450aa1a49350708519586efba8b8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 20 Feb 2025 14:44:50 +0000 Subject: [PATCH 245/281] nats streaming file download: the project/compute server part seems done - the hub express server part is just a minimal proof of concept --- src/packages/hub/proxy/handle-request.ts | 6 +-- src/packages/nats/files/{get.ts => read.ts} | 54 ++++++++++++++++--- src/packages/project/nats/api/index.ts | 5 ++ .../project/nats/browser-websocket-api.ts | 2 +- src/packages/project/nats/files/read.ts | 44 +++++++++++++++ src/packages/project/nats/index.ts | 2 + 6 files changed, 102 insertions(+), 11 deletions(-) rename src/packages/nats/files/{get.ts => read.ts} (75%) create mode 100644 src/packages/project/nats/files/read.ts diff --git a/src/packages/hub/proxy/handle-request.ts b/src/packages/hub/proxy/handle-request.ts index 09854f9302..e2cd8a098a 100644 --- a/src/packages/hub/proxy/handle-request.ts +++ b/src/packages/hub/proxy/handle-request.ts @@ -10,7 +10,7 @@ import { stripBasePath } from "./util"; import { ProjectControlFunction } from "@cocalc/server/projects/control"; import siteUrl from "@cocalc/database/settings/site-url"; import { parseReq } from "./parse"; -import { readFile } from "@cocalc/nats/files/get"; +import { readFile as readProjectFile } from "@cocalc/nats/files/read"; const logger = getLogger("proxy:handle-request"); @@ -80,7 +80,7 @@ export default function init({ projectControl, isPersonal }: Options) { const url = stripBasePath(req.url); // TODO: parseReq is called again in getTarget so need to refactor... const { type, project_id } = parseReq(url, remember_me, api_key); - if (type == "raw") { + if (type == "raw" && !url.includes("primus")) { dbg("handling the request via NATS!"); // TODO: do better, obviously const i = url.indexOf("raw/"); @@ -91,7 +91,7 @@ export default function init({ projectControl, isPersonal }: Options) { const fileName = path; // todo res.setHeader("Content-disposition", "attachment; filename=" + fileName); res.setHeader("Content-type", "text/plain"); // todo - for await (const chunk of await readFile({ + for await (const chunk of await readProjectFile({ project_id, compute_server_id, path, diff --git a/src/packages/nats/files/get.ts b/src/packages/nats/files/read.ts similarity index 75% rename from src/packages/nats/files/get.ts rename to src/packages/nats/files/read.ts index a9374ab89b..a07c02ed53 100644 --- a/src/packages/nats/files/get.ts +++ b/src/packages/nats/files/read.ts @@ -38,13 +38,22 @@ for await (const chunk of await a.readFile({project_id:'00847397-d6a8-4cb0-96a8- import { getEnv } from "@cocalc/nats/client"; import { projectSubject } from "@cocalc/nats/names"; -import { Empty, headers } from "@nats-io/nats-core"; +import { Empty, headers, type Subscription } from "@nats-io/nats-core"; + +let sub: Subscription | null = null; +export async function close() { + if (sub == null) { + return; + } + await sub.drain(); + sub = null; +} function getSubject({ project_id, compute_server_id }) { return projectSubject({ project_id, compute_server_id, - service: "files-get", + service: "files:read", }); } @@ -53,13 +62,23 @@ export async function createServer({ project_id, compute_server_id, }) { + if (sub != null) { + return; + } const { nc } = await getEnv(); const subject = getSubject({ project_id, compute_server_id, }); - console.log(subject); - const sub = nc.subscribe(subject); + // console.log(subject); + sub = nc.subscribe(subject); + listen(createReadStream); +} + +async function listen(createReadStream) { + if (sub == null) { + return; + } for await (const mesg of sub) { try { await handleMessage(mesg, createReadStream); @@ -69,6 +88,7 @@ export async function createServer({ } catch (err) { const h = headers(); h.append("error", `${err}`); + // console.log("sending ERROR", err); mesg.respond(Empty, { headers: h }); } } @@ -79,16 +99,27 @@ async function handleMessage(mesg, createReadStream) { const { path } = jc.decode(mesg.data); let seq = 0; for await (const chunk of createReadStream(path, { - highWaterMark: 16384 * 16 * 2, + highWaterMark: 16384 * 16 * 3, })) { const h = headers(); seq += 1; h.append("seq", `${seq}`); + // console.log("sending ", { seq, bytes: chunk.length }); mesg.respond(chunk, { headers: h }); } } -export async function* readFile({ project_id, compute_server_id, path }) { +export async function* readFile({ + project_id, + compute_server_id, + path, + maxWait = 1000 * 60 * 10, // 10 minutes +}: { + project_id: string; + compute_server_id: number; + path: string; + maxWait?: number; +}) { const { nc, jc } = await getEnv(); const subject = getSubject({ project_id, @@ -96,7 +127,10 @@ export async function* readFile({ project_id, compute_server_id, path }) { }); const v: any = []; let seq = 0; - for await (const resp of await nc.requestMany(subject, jc.encode({ path }))) { + let bytes = 0; + for await (const resp of await nc.requestMany(subject, jc.encode({ path }), { + maxWait, + })) { for (const [key, value] of resp.headers) { if (key == "error") { throw Error(value); @@ -104,6 +138,8 @@ export async function* readFile({ project_id, compute_server_id, path }) { return; } else if (key == "seq") { const next = parseInt(value); + bytes = resp.data.length; + // console.log("received seq", { seq: next, bytes }); if (next != seq + 1) { throw Error("lost data"); } @@ -112,5 +148,9 @@ export async function* readFile({ project_id, compute_server_id, path }) { } yield resp.data; } + if (bytes != 0) { + throw Error("truncated"); + } + // console.log("done!"); return v; } diff --git a/src/packages/project/nats/api/index.ts b/src/packages/project/nats/api/index.ts index c320c7f3c0..407b3beff6 100644 --- a/src/packages/project/nats/api/index.ts +++ b/src/packages/project/nats/api/index.ts @@ -45,6 +45,7 @@ import { terminate as terminateOpenFiles } from "@cocalc/project/nats/open-files import { close as closeListings } from "@cocalc/project/nats/listings"; import { Svcm } from "@nats-io/services"; import { compute_server_id, project_id } from "@cocalc/project/data"; +import { close as closeFilesRead } from "@cocalc/project/nats/files/read"; const logger = getLogger("project:nats:api"); const jc = JSONCodec(); @@ -82,6 +83,10 @@ async function listen(api, subject) { closeListings(); mesg.respond(jc.encode({ status: "terminated", service })); continue; + } else if (service == "files:read") { + await closeFilesRead(); + mesg.respond(jc.encode({ status: "terminated", service })); + continue; } else if (service == "api") { // special hook so admin can terminate handling. This is useful for development. console.warn("TERMINATING listening on ", subject); diff --git a/src/packages/project/nats/browser-websocket-api.ts b/src/packages/project/nats/browser-websocket-api.ts index abc6a9bd13..6465f2eaef 100644 --- a/src/packages/project/nats/browser-websocket-api.ts +++ b/src/packages/project/nats/browser-websocket-api.ts @@ -6,7 +6,7 @@ How to do development (so in a dev project doing cc-in-cc dev): 0. From the browser, send a terminate-handler message, so the handler running in the project stops: - await cc.client.nats_client.projectWebsocketApi({project_id:'56eb622f-d398-489a-83ef-c09f1a1e8094', mesg:{cmd:"terminate"}}) + await cc.client.nats_client.projectWebsocketApi({project_id:cc.current().project_id, mesg:{cmd:"terminate"}}) 1. Open a terminal in the project itself, which sets up the required environment variables, e.g., - COCALC_NATS_JWT -- this has the valid JWT issued to grant the project rights to use nats diff --git a/src/packages/project/nats/files/read.ts b/src/packages/project/nats/files/read.ts new file mode 100644 index 0000000000..eecd77270b --- /dev/null +++ b/src/packages/project/nats/files/read.ts @@ -0,0 +1,44 @@ +/* + +DEVELOPMENT: + + +1. Stop files:read service running in the project by running this in your browser: + + await cc.client.nats_client.projectApi(cc.current()).system.terminate({service:'files:read'}) + + {status: 'terminated', service: 'files:read'} + + +2. Setup the project environment variables. Then start the server in node: + + + ~/cocalc/src/packages/project/nats$ . project-env.sh + $ node + Welcome to Node.js v18.17.1. + Type ".help" for more information. + + require('@cocalc/project/nats/files/read').init() + + +*/ + +import "@cocalc/project/nats/env"; // ensure nats env available + +import { createReadStream as fs_createReadStream } from "fs"; +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { join } from "path"; +import { createServer, close } from "@cocalc/nats/files/read"; +export { close }; + +function createReadStream(path: string) { + if (path[0] != "/" && process.env.HOME) { + path = join(process.env.HOME, path); + } + return fs_createReadStream(path); +} + +// the project should call this on startup: +export async function init() { + await createServer({ project_id, compute_server_id, createReadStream }); +} diff --git a/src/packages/project/nats/index.ts b/src/packages/project/nats/index.ts index 7f2bdaae84..fb4b2f92f7 100644 --- a/src/packages/project/nats/index.ts +++ b/src/packages/project/nats/index.ts @@ -12,6 +12,7 @@ import { init as initOpenFiles } from "./open-files"; // TODO: initWebsocketApi is temporary import { init as initWebsocketApi } from "./browser-websocket-api"; import { init as initListings } from "./listings"; +import { init as initRead } from "./files/read"; const logger = getLogger("project:nats:index"); @@ -21,4 +22,5 @@ export default async function init() { await initOpenFiles(); initWebsocketApi(); await initListings(); + await initRead(); } From fef2012eddd16a326a465ae1370aa1e95cdbbab7 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 20 Feb 2025 15:52:32 +0000 Subject: [PATCH 246/281] nats file download: more integration throughout the system --- src/packages/frontend/client/project.ts | 10 +++++- .../frontend/frame-editors/frame-tree/util.ts | 15 +++++++-- .../frontend/frame-editors/generic/client.ts | 12 +++++-- src/packages/frontend/lib/raw-url.ts | 15 +++++++-- src/packages/frontend/project_store.ts | 9 ++++-- src/packages/hub/package.json | 1 + src/packages/hub/proxy/handle-request.ts | 32 ++++++++++++------- src/packages/hub/proxy/parse.ts | 10 +++--- src/packages/pnpm-lock.yaml | 3 ++ src/packages/project/nats/files/read.ts | 1 + 10 files changed, 81 insertions(+), 27 deletions(-) diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index 9182636ec2..05da8af05c 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -106,13 +106,21 @@ export class ProjectClient { public read_file(opts: { project_id: string; // string or array of strings path: string; // string or array of strings + compute_server_id?: number; }): string { const base_path = appBasePath; if (opts.path[0] === "/") { // absolute path to the root opts.path = HOME_ROOT + opts.path; // use root symlink, which is created by start_smc } - return encode_path(join(base_path, `${opts.project_id}/raw/${opts.path}`)); + let url = join( + base_path, + `${opts.project_id}/files/${encode_path(opts.path)}`, + ); + if (opts.compute_server_id) { + url += `?id=${opts.compute_server_id}`; + } + return url; } public async copy_path_between_projects(opts: { diff --git a/src/packages/frontend/frame-editors/frame-tree/util.ts b/src/packages/frontend/frame-editors/frame-tree/util.ts index 0164bf7b86..bee5532b73 100644 --- a/src/packages/frontend/frame-editors/frame-tree/util.ts +++ b/src/packages/frontend/frame-editors/frame-tree/util.ts @@ -22,10 +22,19 @@ export function parse_path(path: string): { return { directory: x.head, base: y.name, filename: x.tail }; } -export function raw_url(project_id: string, path: string): string { +export function raw_url( + project_id: string, + path: string, + // [ ] todo: make this required and explicitly called everywhere + compute_server_id?: number, +): string { // we have to encode the path, since we query this raw server. see // https://github.com/sagemathinc/cocalc/issues/5542 - // but actually, this is a problem for types of files, not just PDF + // but actually, this is a problem for all types of files, not just PDF const path_enc = encode_path(path); - return join(appBasePath, project_id, "raw", path_enc); + let url = join(appBasePath, project_id, "files", path_enc); + if (compute_server_id) { + url += `?id=${compute_server_id}`; + } + return url; } diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index 95a597dd5e..0457eed620 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -274,6 +274,14 @@ export async function project_api(project_id: string): Promise { } // Returns the raw URL to read the file from the project. -export function raw_url_of_file(project_id: string, path: string): string { - return webapp_client.project_client.read_file({ project_id, path }); +export function raw_url_of_file( + project_id: string, + path: string, + compute_server_id?: number, +): string { + return webapp_client.project_client.read_file({ + project_id, + path, + compute_server_id, + }); } diff --git a/src/packages/frontend/lib/raw-url.ts b/src/packages/frontend/lib/raw-url.ts index 8287c85cb0..de0f92f7a0 100644 --- a/src/packages/frontend/lib/raw-url.ts +++ b/src/packages/frontend/lib/raw-url.ts @@ -1,7 +1,7 @@ /* The raw URL is the following, of course encoded as a URL: -.../{project_id}/raw/{full relative path in the project to file} +.../{project_id}/files/{full relative path in the project to file}?compute_server_id=[global id number] */ import { join } from "path"; @@ -10,10 +10,19 @@ import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; interface Options { project_id: string; path: string; + compute_server_id?: number; } -export default function rawURL({ project_id, path }: Options): string { - return join(appBasePath, project_id, "raw", encodePath(path)); +export default function rawURL({ + project_id, + path, + compute_server_id, +}: Options): string { + let url = join(appBasePath, project_id, "files", encodePath(path)); + if (compute_server_id) { + url += `?id=${compute_server_id}`; + } + return url; } export function encodePath(path: string) { diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 26179a0ec1..757351d8ca 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -556,10 +556,15 @@ export class ProjectStore extends Store { }; }; - get_raw_link = (path) => { + get_raw_link = (path, compute_server_id?: number) => { let url = document.URL; url = url.slice(0, url.indexOf("/projects/")); - return `${url}/${this.project_id}/raw/${misc.encode_path(path)}`; + url = `${url}/${this.project_id}/files/${misc.encode_path(path)}`; + const computeServerId = compute_server_id ?? this.get("compute_server_id"); + if (computeServerId) { + url += `&id=${computeServerId}`; + } + return url; }; // returns false, if this project isn't capable of opening a file with the given extension diff --git a/src/packages/hub/package.json b/src/packages/hub/package.json index 311e63f1cb..902d750289 100644 --- a/src/packages/hub/package.json +++ b/src/packages/hub/package.json @@ -46,6 +46,7 @@ "lodash": "^4.17.21", "lru-cache": "^7.18.3", "mime": "^1.3.4", + "mime-types": "^2.1.35", "mkdirp": "^1.0.4", "ms": "2.1.2", "nats": "^2.29.1", diff --git a/src/packages/hub/proxy/handle-request.ts b/src/packages/hub/proxy/handle-request.ts index e2cd8a098a..5baa6bf4fa 100644 --- a/src/packages/hub/proxy/handle-request.ts +++ b/src/packages/hub/proxy/handle-request.ts @@ -11,6 +11,9 @@ import { ProjectControlFunction } from "@cocalc/server/projects/control"; import siteUrl from "@cocalc/database/settings/site-url"; import { parseReq } from "./parse"; import { readFile as readProjectFile } from "@cocalc/nats/files/read"; +import { path_split } from "@cocalc/util/misc"; +import { once } from "@cocalc/util/async-utils"; +import mime from "mime-types"; const logger = getLogger("proxy:handle-request"); @@ -46,7 +49,8 @@ export default function init({ projectControl, isPersonal }: Options) { logger.silly(req.url, ...args); }; dbg("got request"); - dbg("headers = ", req.headers); + // dangerous/verbose to log...? + // dbg("headers = ", req.headers); if (!isPersonal && versionCheckFails(req, res)) { dbg("version check failed"); @@ -80,23 +84,29 @@ export default function init({ projectControl, isPersonal }: Options) { const url = stripBasePath(req.url); // TODO: parseReq is called again in getTarget so need to refactor... const { type, project_id } = parseReq(url, remember_me, api_key); - if (type == "raw" && !url.includes("primus")) { + if (type == "files") { + // TODO: auth! dbg("handling the request via NATS!"); - // TODO: do better, obviously - const i = url.indexOf("raw/"); - const compute_server_id = 0; - const path = url.slice(i + 4); - // worry decodeURI - dbg("NATs: get", { project_id, path, compute_server_id }); - const fileName = path; // todo + const i = url.indexOf("files/"); + const compute_server_id = req.query.id ?? 0; + let j = url.lastIndexOf("?"); + if (j == -1) { + j = url.length; + } + const path = decodeURIComponent(url.slice(i + "files/".length, j)); + dbg("NATs: get", { project_id, path, compute_server_id, url }); + const fileName = path_split(path).tail; res.setHeader("Content-disposition", "attachment; filename=" + fileName); - res.setHeader("Content-type", "text/plain"); // todo + res.setHeader("Content-type", mime.lookup(fileName)); for await (const chunk of await readProjectFile({ project_id, compute_server_id, path, })) { - res.write(chunk); + if (!res.write(chunk)) { + // backpressure -- wait for it to resolve + await once(res, "drain"); + } } res.end(); return; diff --git a/src/packages/hub/proxy/parse.ts b/src/packages/hub/proxy/parse.ts index 4acf792cec..43f97cb342 100644 --- a/src/packages/hub/proxy/parse.ts +++ b/src/packages/hub/proxy/parse.ts @@ -1,9 +1,9 @@ -type ProxyType = "port" | "raw" | "server"; +type ProxyType = "port" | "raw" | "server" | "files"; export function parseReq( url: string, // with base_path removed (url does start with /) remember_me?: string, // only impacts the key that is returned - api_key?: string // only impacts key + api_key?: string, // only impacts key ): { key: string; // used for caching type: ProxyType; @@ -16,15 +16,15 @@ export function parseReq( } const v = url.split("/").slice(1); const project_id = v[0]; - if (v[1] != "port" && v[1] != "raw" && v[1] != "server") { + if (v[1] != "port" && v[1] != "raw" && v[1] != "server" && v[1] != "files") { throw Error( - `invalid type -- "${v[1]}" must be "port", "raw" or "server" in url="${url}"` + `invalid type -- "${v[1]}" must be "port", "raw", "files" or "server" in url="${url}"`, ); } const type: ProxyType = v[1]; let internal_url: string | undefined = undefined; let port_desc: string; - if (type == "raw") { + if (type == "raw" || type == "files") { port_desc = ""; } else if (type === "port") { port_desc = v[2]; diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index a5fdd78e69..d21c0965dd 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -854,6 +854,9 @@ importers: mime: specifier: ^1.3.4 version: 1.6.0 + mime-types: + specifier: ^2.1.35 + version: 2.1.35 mkdirp: specifier: ^1.0.4 version: 1.0.4 diff --git a/src/packages/project/nats/files/read.ts b/src/packages/project/nats/files/read.ts index eecd77270b..38de6610de 100644 --- a/src/packages/project/nats/files/read.ts +++ b/src/packages/project/nats/files/read.ts @@ -9,6 +9,7 @@ DEVELOPMENT: {status: 'terminated', service: 'files:read'} +You can also skip step 1 if you instead set COMPUTE_SERVER_ID to something nonzero... 2. Setup the project environment variables. Then start the server in node: From 70e75a982f3690f2d9205bf891d92c5fddd4e78c Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 20 Feb 2025 22:55:41 +0000 Subject: [PATCH 247/281] nats file read -- add auth --- .../hub/proxy/check-for-access-to-project.ts | 4 ++-- src/packages/hub/proxy/handle-request.ts | 19 ++++++++++++++++--- src/packages/hub/proxy/target.ts | 9 ++++----- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/packages/hub/proxy/check-for-access-to-project.ts b/src/packages/hub/proxy/check-for-access-to-project.ts index be1f128985..b9745af430 100644 --- a/src/packages/hub/proxy/check-for-access-to-project.ts +++ b/src/packages/hub/proxy/check-for-access-to-project.ts @@ -24,9 +24,9 @@ interface Options { } // 1 minute cache: grant "yes" for a while -const yesCache = new LRU({ max: 20000, ttl: 1000 * 60 * 1 }); +const yesCache = new LRU({ max: 20000, ttl: 1000 * 60 * 1.5 }); // 5 second cache: recheck "no" much more frequently -const noCache = new LRU({ max: 20000, ttl: 1000 * 5 }); +const noCache = new LRU({ max: 20000, ttl: 1000 * 15 }); export default async function hasAccess(opts: Options): Promise { if (opts.isPersonal) { diff --git a/src/packages/hub/proxy/handle-request.ts b/src/packages/hub/proxy/handle-request.ts index 5baa6bf4fa..cc80750f2f 100644 --- a/src/packages/hub/proxy/handle-request.ts +++ b/src/packages/hub/proxy/handle-request.ts @@ -13,6 +13,7 @@ import { parseReq } from "./parse"; import { readFile as readProjectFile } from "@cocalc/nats/files/read"; import { path_split } from "@cocalc/util/misc"; import { once } from "@cocalc/util/async-utils"; +import hasAccess from "./check-for-access-to-project"; import mime from "mime-types"; const logger = getLogger("proxy:handle-request"); @@ -82,11 +83,22 @@ export default function init({ projectControl, isPersonal }: Options) { } const url = stripBasePath(req.url); + const parsed = parseReq(url, remember_me, api_key); // TODO: parseReq is called again in getTarget so need to refactor... - const { type, project_id } = parseReq(url, remember_me, api_key); + const { type, project_id } = parsed; if (type == "files") { - // TODO: auth! - dbg("handling the request via NATS!"); + dbg("handling the request via nats"); + if ( + !(await hasAccess({ + project_id, + remember_me, + api_key, + type: "read", + isPersonal, + })) + ) { + throw Error(`user does not have read access to project`); + } const i = url.indexOf("files/"); const compute_server_id = req.query.id ?? 0; let j = url.lastIndexOf("?"); @@ -118,6 +130,7 @@ export default function init({ projectControl, isPersonal }: Options) { url, isPersonal, projectControl, + parsed, }); // It's http here because we've already got past the ssl layer. This is all internal. diff --git a/src/packages/hub/proxy/target.ts b/src/packages/hub/proxy/target.ts index 3a88936e57..3f8537ad06 100644 --- a/src/packages/hub/proxy/target.ts +++ b/src/packages/hub/proxy/target.ts @@ -43,6 +43,7 @@ interface Options { url: string; isPersonal: boolean; projectControl: ProjectControlFunction; + parsed?: ReturnType; } export async function getTarget({ @@ -51,16 +52,14 @@ export async function getTarget({ url, isPersonal, projectControl, + parsed, }: Options): Promise<{ host: string; port: number; internal_url: string | undefined; }> { - const { key, type, project_id, port_desc, internal_url } = parseReq( - url, - remember_me, - api_key, - ); + const { key, type, project_id, port_desc, internal_url } = + parsed ?? parseReq(url, remember_me, api_key); if (cache.has(key)) { return cache.get(key) as any; From 1465dd437be29e4be003dcb775b47ea4e9d056c4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 20 Feb 2025 23:23:11 +0000 Subject: [PATCH 248/281] "raw url" --> "files url" for download -- refactor code so url always computed via same function (instead of in four different places numerous ways) --- .../frontend/frame-editors/frame-tree/util.ts | 15 ++------ .../frontend/frame-editors/generic/client.ts | 13 ------- .../slideshow-revealjs/nbconvert.ts | 8 +++-- src/packages/frontend/jupyter/nbconvert.tsx | 2 +- src/packages/frontend/lib/cocalc-urls.ts | 20 +++++++++++ src/packages/frontend/lib/raw-url.ts | 35 ------------------- .../frontend/project/explorer/download.tsx | 2 +- .../frontend/project/page/url-transform.ts | 8 ++--- src/packages/frontend/project_store.ts | 17 ++++----- src/packages/jupyter/redux/store.ts | 6 ---- 10 files changed, 40 insertions(+), 86 deletions(-) delete mode 100644 src/packages/frontend/lib/raw-url.ts diff --git a/src/packages/frontend/frame-editors/frame-tree/util.ts b/src/packages/frontend/frame-editors/frame-tree/util.ts index bee5532b73..f49098ee3a 100644 --- a/src/packages/frontend/frame-editors/frame-tree/util.ts +++ b/src/packages/frontend/frame-editors/frame-tree/util.ts @@ -8,9 +8,7 @@ Utility functions useful for frame-tree editors. */ import { path_split, separate_file_extension } from "@cocalc/util/misc"; -import { encode_path } from "@cocalc/util/misc"; -import { join } from "path"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; +import { fileURL } from "@cocalc/frontend/lib/cocalc-urls"; export function parse_path(path: string): { directory: string; @@ -25,16 +23,7 @@ export function parse_path(path: string): { export function raw_url( project_id: string, path: string, - // [ ] todo: make this required and explicitly called everywhere compute_server_id?: number, ): string { - // we have to encode the path, since we query this raw server. see - // https://github.com/sagemathinc/cocalc/issues/5542 - // but actually, this is a problem for all types of files, not just PDF - const path_enc = encode_path(path); - let url = join(appBasePath, project_id, "files", path_enc); - if (compute_server_id) { - url += `?id=${compute_server_id}`; - } - return url; + return fileURL({ project_id, path, compute_server_id }); } diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index 0457eed620..e5c96ef9c8 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -272,16 +272,3 @@ import { API } from "@cocalc/frontend/project/websocket/api"; export async function project_api(project_id: string): Promise { return (await project_websocket(project_id)).api as API; } - -// Returns the raw URL to read the file from the project. -export function raw_url_of_file( - project_id: string, - path: string, - compute_server_id?: number, -): string { - return webapp_client.project_client.read_file({ - project_id, - path, - compute_server_id, - }); -} diff --git a/src/packages/frontend/frame-editors/jupyter-editor/slideshow-revealjs/nbconvert.ts b/src/packages/frontend/frame-editors/jupyter-editor/slideshow-revealjs/nbconvert.ts index 6c3ee8a187..5ee2318990 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/slideshow-revealjs/nbconvert.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/slideshow-revealjs/nbconvert.ts @@ -4,12 +4,14 @@ */ import { path_split, separate_file_extension } from "@cocalc/util/misc"; -import { exec, raw_url_of_file } from "../../generic/client"; +import { exec } from "../../generic/client"; +import { fileURL } from "@cocalc/frontend/lib/cocalc-urls"; import { sanitize_nbconvert_path } from "@cocalc/util/sanitize-nbconvert"; export async function revealjs_slideshow_html( project_id: string, - path: string + path: string, + compute_server_id?: number, ): Promise { const split = path_split(path); // The _ bewlo is because of https://github.com/sagemathinc/cocalc/issues/4066, i.e., otherwise @@ -34,5 +36,5 @@ export async function revealjs_slideshow_html( const html_filename = split.head ? [split.head, base + ext].join("/") : base + ext; - return raw_url_of_file(project_id, html_filename); + return fileURL({ project_id, path: html_filename, compute_server_id }); } diff --git a/src/packages/frontend/jupyter/nbconvert.tsx b/src/packages/frontend/jupyter/nbconvert.tsx index 7b3aa82516..9ba6ab3bc8 100644 --- a/src/packages/frontend/jupyter/nbconvert.tsx +++ b/src/packages/frontend/jupyter/nbconvert.tsx @@ -196,7 +196,7 @@ export const NBConvert: React.FC = React.memo( ext = info.ext; } const targetPath = misc.change_filename_extension(path, ext); - const url = actions.store.get_raw_link(targetPath); + const url = actions.store.fileURL(targetPath); return { targetPath, url, info }; } diff --git a/src/packages/frontend/lib/cocalc-urls.ts b/src/packages/frontend/lib/cocalc-urls.ts index 1d14952693..8386032122 100644 --- a/src/packages/frontend/lib/cocalc-urls.ts +++ b/src/packages/frontend/lib/cocalc-urls.ts @@ -3,6 +3,26 @@ import { is_valid_uuid_string as isUUID } from "@cocalc/util/misc"; import { splitFirst } from "@cocalc/util/misc"; import Fragment, { FragmentId } from "@cocalc/frontend/misc/fragment-id"; import { APP_ROUTES } from "@cocalc/util/routing/app"; +import { join } from "path"; +import { encode_path } from "@cocalc/util/misc"; + +// URL to use http to download a file from a project or compute server +// that you collaborate on. +export function fileURL({ + project_id, + compute_server_id, + path, +}: { + project_id: string; + path: string; + compute_server_id?: number; +}): string { + let url = join(appBasePath, project_id, "files", encode_path(path)); + if (compute_server_id) { + url += `?id=${compute_server_id}`; + } + return url; +} function getOrigin(): string { // This is a situation where our choice of definition of "/" for the diff --git a/src/packages/frontend/lib/raw-url.ts b/src/packages/frontend/lib/raw-url.ts deleted file mode 100644 index de0f92f7a0..0000000000 --- a/src/packages/frontend/lib/raw-url.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* -The raw URL is the following, of course encoded as a URL: - -.../{project_id}/files/{full relative path in the project to file}?compute_server_id=[global id number] -*/ - -import { join } from "path"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; - -interface Options { - project_id: string; - path: string; - compute_server_id?: number; -} - -export default function rawURL({ - project_id, - path, - compute_server_id, -}: Options): string { - let url = join(appBasePath, project_id, "files", encodePath(path)); - if (compute_server_id) { - url += `?id=${compute_server_id}`; - } - return url; -} - -export function encodePath(path: string) { - const segments = path.split("/"); - const encoded: string[] = []; - for (const segment of segments) { - encoded.push(encodeURIComponent(segment)); - } - return encoded.join("/"); -} diff --git a/src/packages/frontend/project/explorer/download.tsx b/src/packages/frontend/project/explorer/download.tsx index 7acb938c4a..f3aab57d65 100644 --- a/src/packages/frontend/project/explorer/download.tsx +++ b/src/packages/frontend/project/explorer/download.tsx @@ -47,7 +47,7 @@ export default function Download({}) { setArchiveMode(!!isdir); if (!isdir) { const store = actions?.get_store(); - setUrl(store?.get_raw_link(file) ?? ""); + setUrl(store?.fileURL(file) ?? ""); } }, [checked_files, current_path]); diff --git a/src/packages/frontend/project/page/url-transform.ts b/src/packages/frontend/project/page/url-transform.ts index 2eb555c6cf..1bb2a16758 100644 --- a/src/packages/frontend/project/page/url-transform.ts +++ b/src/packages/frontend/project/page/url-transform.ts @@ -1,5 +1,5 @@ import { join } from "path"; -import rawURL from "@cocalc/frontend/lib/raw-url"; +import { fileURL } from "@cocalc/frontend/lib/cocalc-urls"; import { containingPath } from "@cocalc/util/misc"; import { isCoCalcURL, parseCoCalcURL } from "@cocalc/frontend/lib/cocalc-urls"; @@ -12,18 +12,18 @@ interface Options { export default function getUrlTransform({ project_id, path }: Options) { const dir = containingPath(path); return (href: string, tag: string) => { - if(href.startsWith('data:')) return; // never change data: urls in any way. + if (href.startsWith("data:")) return; // never change data: urls in any way. if (tag == "a" || href.includes("://")) { // Anchor tags are dealt with via AnchorTagComponent // We also only modify local urls and cloud urls (only on frontend -- they will fail on share server). if (isCoCalcURL(href)) { const { project_id, path } = parseCoCalcURL(href); if (project_id != null && path != null) { - return rawURL({ project_id, path }); + return fileURL({ project_id, path }); } } return; } - return rawURL({ project_id, path: join(dir, href) }); + return fileURL({ project_id, path: join(dir, href) }); }; } diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 757351d8ca..d60f3a56a5 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -55,6 +55,7 @@ import { } from "./project/page/flyouts/utils"; import { get_local_storage } from "@cocalc/frontend/misc"; import { QueryParams } from "@cocalc/frontend/misc/query-params"; +import { fileURL } from "@cocalc/frontend/lib/cocalc-urls"; export { FILE_ACTIONS as file_actions, ProjectActions }; @@ -556,16 +557,12 @@ export class ProjectStore extends Store { }; }; - get_raw_link = (path, compute_server_id?: number) => { - let url = document.URL; - url = url.slice(0, url.indexOf("/projects/")); - url = `${url}/${this.project_id}/files/${misc.encode_path(path)}`; - const computeServerId = compute_server_id ?? this.get("compute_server_id"); - if (computeServerId) { - url += `&id=${computeServerId}`; - } - return url; - }; + fileURL = (path, compute_server_id?: number) => { + return fileURL({ + project_id: this.project_id, + path, + compute_server_id: compute_server_id ?? this.get("compute_server_id"), + }); }; // returns false, if this project isn't capable of opening a file with the given extension async can_open_file_ext( diff --git a/src/packages/jupyter/redux/store.ts b/src/packages/jupyter/redux/store.ts index 82525e3f03..8c2eb3f343 100644 --- a/src/packages/jupyter/redux/store.ts +++ b/src/packages/jupyter/redux/store.ts @@ -375,12 +375,6 @@ export class JupyterStore extends Store { return get_kernel_selection(kernels); }; - get_raw_link = (path: any) => { - return this.redux - .getProjectStore(this.get("project_id")) - .get_raw_link(path); - }; - // NOTE: defaults for these happen to be true if not given (due to bad // choice of name by some extension author). public is_cell_editable(id: string): boolean { From ec945332c1ca12f0fe5e7837ab8627278132c51e Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 21 Feb 2025 01:44:05 +0000 Subject: [PATCH 249/281] nats: create streaming file write service (no testing) - I'm sure this is far from actually working. --- src/packages/nats/files/read.ts | 71 ++++++----- src/packages/nats/files/write.ts | 158 ++++++++++++++++++++++++ src/packages/project/nats/files/read.ts | 10 +- 3 files changed, 208 insertions(+), 31 deletions(-) create mode 100644 src/packages/nats/files/write.ts diff --git a/src/packages/nats/files/read.ts b/src/packages/nats/files/read.ts index a07c02ed53..c448b070b2 100644 --- a/src/packages/nats/files/read.ts +++ b/src/packages/nats/files/read.ts @@ -40,20 +40,22 @@ import { getEnv } from "@cocalc/nats/client"; import { projectSubject } from "@cocalc/nats/names"; import { Empty, headers, type Subscription } from "@nats-io/nats-core"; -let sub: Subscription | null = null; -export async function close() { - if (sub == null) { +let subs: { [name: string]: Subscription } = {}; +export async function close({ project_id, compute_server_id, name = "" }) { + const key = getSubject({ project_id, compute_server_id, name }); + if (subs[key] == null) { return; } + const sub = subs[key]; + delete subs[key]; await sub.drain(); - sub = null; } -function getSubject({ project_id, compute_server_id }) { +function getSubject({ project_id, compute_server_id, name = "" }) { return projectSubject({ project_id, compute_server_id, - service: "files:read", + service: `files:read${name ?? ""}`, }); } @@ -61,40 +63,48 @@ export async function createServer({ createReadStream, project_id, compute_server_id, + name = "", }) { - if (sub != null) { - return; - } - const { nc } = await getEnv(); const subject = getSubject({ project_id, compute_server_id, + name, }); + if (subs[subject] != null) { + return; + } + const { nc } = await getEnv(); // console.log(subject); - sub = nc.subscribe(subject); - listen(createReadStream); + const sub = nc.subscribe(subject); + subs[subject] = sub; + listen(sub, createReadStream); } -async function listen(createReadStream) { - if (sub == null) { - return; - } +async function listen(sub, createReadStream) { + // NOTE: we just handle as many messages as we get in parallel, so this + // could be a large number of simultaneous downloads. These are all by + // authenticated users of the project, and the load is on the project, + // so I think that makes sense. for await (const mesg of sub) { - try { - await handleMessage(mesg, createReadStream); - const h = headers(); - h.append("done", ""); - mesg.respond(Empty, { headers: h }); - } catch (err) { - const h = headers(); - h.append("error", `${err}`); - // console.log("sending ERROR", err); - mesg.respond(Empty, { headers: h }); - } + handleMessage(mesg, createReadStream); } } async function handleMessage(mesg, createReadStream) { + try { + await sendData(mesg, createReadStream); + const h = headers(); + h.append("done", ""); + mesg.respond(Empty, { headers: h }); + } catch (err) { + const h = headers(); + h.append("error", `${err}`); + // console.log("sending ERROR", err); + mesg.respond(Empty, { headers: h }); + } +} + +async function sendData(mesg, createReadStream) { const { jc } = await getEnv(); const { path } = jc.decode(mesg.data); let seq = 0; @@ -111,19 +121,22 @@ async function handleMessage(mesg, createReadStream) { export async function* readFile({ project_id, - compute_server_id, + compute_server_id = 0, path, + name = "", maxWait = 1000 * 60 * 10, // 10 minutes }: { project_id: string; - compute_server_id: number; + compute_server_id?: number; path: string; + name?: string; maxWait?: number; }) { const { nc, jc } = await getEnv(); const subject = getSubject({ project_id, compute_server_id, + name, }); const v: any = []; let seq = 0; diff --git a/src/packages/nats/files/write.ts b/src/packages/nats/files/write.ts new file mode 100644 index 0000000000..c88caea91b --- /dev/null +++ b/src/packages/nats/files/write.ts @@ -0,0 +1,158 @@ +/* +Streaming write over NATS to a project or compute server. + +This is a key component to support user uploads, while being memory efficient +by streaming the write. + +Here's how this works from the side of the compute server: + +- We start a request/response NATS server on the compute server: +- There's one message it accepts, which is: + "Using streaming download to get {path} from [subject]." + The sender of that message should set a long timeout (e.g., 10 minutes). +- It uses the streaming read functionality (in read.ts) to download and write + to disk the file {path}. +- When done it responds {status:"success"} or {status:'error', error:'message...'} + +Here's how it works from the side of whoever is sending the file: + +- Start read server at [subject] that can send {path}. +- Send a request saying "we are making {path} available to you at [subject]." +- Get back "ok" or error. On error (or timeout), close the read server. +- Serve {path} exactly once using the server. When finish sending {path}, + close it and clean up. We're done. + +*/ + +import { getEnv } from "@cocalc/nats/client"; +import { readFile } from "./read"; +import { randomId } from "@cocalc/nats/names"; +import { + close as closeReadService, + createServer as createReadServer, +} from "./read"; +import { projectSubject } from "@cocalc/nats/names"; +import { type Subscription } from "@nats-io/nats-core"; +import { type Readable } from "node:stream"; + +function getWriteSubject({ project_id, compute_server_id }) { + return projectSubject({ + project_id, + compute_server_id, + service: "files:write", + }); +} + +let sub: Subscription | null = null; +export async function close() { + if (sub == null) { + return; + } + await sub.drain(); + sub = null; +} + +export async function createServer({ + project_id, + compute_server_id, + createWriteStream, +}) { + if (sub != null) { + return; + } + const { nc } = await getEnv(); + const subject = getWriteSubject({ project_id, compute_server_id }); + sub = nc.subscribe(subject); + listen({ createWriteStream, project_id, compute_server_id }); +} + +async function listen({ createWriteStream, project_id, compute_server_id }) { + if (sub == null) { + return; + } + // NOTE: we just handle as many messages as we get in parallel, so this + // could be a large number of simultaneous downloads. These are all by + // authenticated users of the project, and the load is on the project, + // so I think that makes sense. + for await (const mesg of sub) { + handleMessage({ mesg, createWriteStream, project_id, compute_server_id }); + } +} + +async function handleMessage({ + mesg, + createWriteStream, + project_id, + compute_server_id, +}) { + let error = ""; + try { + const { jc } = await getEnv(); + const { path, name, maxWait } = jc.decode(mesg.data); + const writeStream = createWriteStream(path); + writeStream.on("error", (err) => { + error = `${err}`; + mesg.respond({ error, status: "error" }); + console.warn(`error writing ${path}: ${error}`); + }); + for await (const chunk of await readFile({ + project_id, + compute_server_id, + name, + path, + maxWait, + })) { + if (error) { + return; + } + writeStream.write(chunk); + } + writeStream.end(); + mesg.respond({ status: "success" }); + } catch (err) { + if (!error) { + mesg.respond({ error: `${err}`, status: "error" }); + } + } +} + +export async function writeFile({ + project_id, + compute_server_id = 0, + path, + stream, + maxWait = 1000 * 60 * 10, // 10 minutes +}: { + project_id: string; + compute_server_id?: number; + path: string; + stream: Readable; + maxWait?: number; +}): Promise { + const name = randomId(); + try { + function createReadStream() { + return stream; + } + // start read server + await createReadServer({ + createReadStream, + project_id, + compute_server_id, + name, + }); + // tell compute server to start reading our file. + const { nc, jc } = await getEnv(); + const resp = await nc.request( + getWriteSubject({ project_id, compute_server_id }), + jc.encode({ name, path, maxWait }), + { timeout: maxWait }, + ); + const { error } = jc.decode(resp.data); + if (error) { + throw Error(error); + } + } finally { + await closeReadService({ project_id, compute_server_id, name }); + } +} diff --git a/src/packages/project/nats/files/read.ts b/src/packages/project/nats/files/read.ts index 38de6610de..76882f2fd0 100644 --- a/src/packages/project/nats/files/read.ts +++ b/src/packages/project/nats/files/read.ts @@ -29,8 +29,10 @@ import "@cocalc/project/nats/env"; // ensure nats env available import { createReadStream as fs_createReadStream } from "fs"; import { compute_server_id, project_id } from "@cocalc/project/data"; import { join } from "path"; -import { createServer, close } from "@cocalc/nats/files/read"; -export { close }; +import { + createServer, + close as closeReadServer, +} from "@cocalc/nats/files/read"; function createReadStream(path: string) { if (path[0] != "/" && process.env.HOME) { @@ -43,3 +45,7 @@ function createReadStream(path: string) { export async function init() { await createServer({ project_id, compute_server_id, createReadStream }); } + +export async function close() { + await closeReadServer({ project_id, compute_server_id }); +} From 5be29e1a424c7d12abb3912f1ef855463dff6d7b Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 21 Feb 2025 02:20:34 +0000 Subject: [PATCH 250/281] nats file write: add example and fix the one (and only?) mistake --- src/packages/nats/files/read.ts | 2 +- src/packages/nats/files/write.ts | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/packages/nats/files/read.ts b/src/packages/nats/files/read.ts index c448b070b2..966618afe6 100644 --- a/src/packages/nats/files/read.ts +++ b/src/packages/nats/files/read.ts @@ -26,7 +26,7 @@ DEVELOPMENT: ~/cocalc/src/packages/backend$ node -require('@cocalc/backend/nats'); a = require('@cocalc/nats/files/get'); a.createServer({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,createReadStream:require('fs').createReadStream}) +require('@cocalc/backend/nats'); a = require('@cocalc/nats/files/read'); a.createServer({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,createReadStream:require('fs').createReadStream}) for await (const chunk of await a.readFile({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,path:'/tmp/a.py'})) { console.log({chunk}); } diff --git a/src/packages/nats/files/write.ts b/src/packages/nats/files/write.ts index c88caea91b..2e4c54496f 100644 --- a/src/packages/nats/files/write.ts +++ b/src/packages/nats/files/write.ts @@ -22,6 +22,19 @@ Here's how it works from the side of whoever is sending the file: - Serve {path} exactly once using the server. When finish sending {path}, close it and clean up. We're done. + + +DEVELOPMENT: + +~/cocalc/src/packages/backend$ node + +require('@cocalc/backend/nats'); a = require('@cocalc/nats/files/write'); + +project_id = '00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'; compute_server_id = 0; await a.createServer({project_id,compute_server_id,createWriteStream:require('fs').createWriteStream}); + +stream=require('fs').createReadStream('env.ts'); +await a.writeFile({stream, project_id, compute_server_id, path:'/tmp/a.ts'}) + */ import { getEnv } from "@cocalc/nats/client"; @@ -92,7 +105,7 @@ async function handleMessage({ const writeStream = createWriteStream(path); writeStream.on("error", (err) => { error = `${err}`; - mesg.respond({ error, status: "error" }); + mesg.respond(jc.encode({ error, status: "error" })); console.warn(`error writing ${path}: ${error}`); }); for await (const chunk of await readFile({ @@ -108,10 +121,10 @@ async function handleMessage({ writeStream.write(chunk); } writeStream.end(); - mesg.respond({ status: "success" }); + mesg.respond(jc.encode({ status: "success" })); } catch (err) { if (!error) { - mesg.respond({ error: `${err}`, status: "error" }); + mesg.respond(jc.encode({ error: `${err}`, status: "error" })); } } } From c2b84be61930d992a43c43215f6f009789e1e621 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 21 Feb 2025 03:08:09 +0000 Subject: [PATCH 251/281] nats unit tests: reorg sync; write new tests for time and writeFile --- .../backend/nats/test/files/write.test.ts | 121 ++++++++++++++++++ .../backend/nats/test/{ => sync}/dko.test.ts | 0 .../nats/test/{ => sync}/dkv-merge.test.ts | 0 .../backend/nats/test/{ => sync}/dkv.test.ts | 0 .../nats/test/{ => sync}/dstream.test.ts | 0 .../nats/test/{ => sync}/open-files.test.ts | 0 src/packages/backend/nats/test/time.test.ts | 30 +++++ src/packages/backend/package.json | 13 +- src/packages/nats/files/write.ts | 39 ++++-- src/packages/nats/time.ts | 2 +- src/packages/pnpm-lock.yaml | 16 +++ 11 files changed, 204 insertions(+), 17 deletions(-) create mode 100644 src/packages/backend/nats/test/files/write.test.ts rename src/packages/backend/nats/test/{ => sync}/dko.test.ts (100%) rename src/packages/backend/nats/test/{ => sync}/dkv-merge.test.ts (100%) rename src/packages/backend/nats/test/{ => sync}/dkv.test.ts (100%) rename src/packages/backend/nats/test/{ => sync}/dstream.test.ts (100%) rename src/packages/backend/nats/test/{ => sync}/open-files.test.ts (100%) create mode 100644 src/packages/backend/nats/test/time.test.ts diff --git a/src/packages/backend/nats/test/files/write.test.ts b/src/packages/backend/nats/test/files/write.test.ts new file mode 100644 index 0000000000..c51a8ad0d3 --- /dev/null +++ b/src/packages/backend/nats/test/files/write.test.ts @@ -0,0 +1,121 @@ +/* +Test async streaming writing of files to compute servers using NATS. + + +DEVELOPMENT: + +pnpm exec jest --watch --forceExit --detectOpenHandles "write.test.ts" +*/ + +import "@cocalc/backend/nats"; +import { close, createServer, writeFile } from "@cocalc/nats/files/write"; +import { createWriteStream, createReadStream } from "fs"; +import { file as tempFile } from "tmp-promise"; +import { writeFile as fsWriteFile, readFile } from "fs/promises"; + +describe("do a basic test that the file writing service works", () => { + const project_id = "00000000-0000-4000-8000-000000000000"; + const compute_server_id = 0; + it("create the write server", async () => { + await createServer({ + project_id, + compute_server_id, + createWriteStream, + }); + }); + + let cleanups: any[] = []; + const CONTENT = "cocalc"; + let source; + it("creates the file we will read", async () => { + const { path, cleanup } = await tempFile(); + source = path; + await fsWriteFile(path, CONTENT); + cleanups.push(cleanup); + }); + + let dest; + it("write to a new file", async () => { + const { path, cleanup } = await tempFile(); + dest = path; + cleanups.push(cleanup); + + const stream = createReadStream(source); + const { bytes, chunks } = await writeFile({ + stream, + project_id, + compute_server_id, + path, + }); + expect(chunks).toBe(1); + expect(bytes).toBe(CONTENT.length); + }); + + it("confirm that the dest file is correct", async () => { + const d = (await readFile(dest)).toString(); + expect(d).toEqual(CONTENT); + }); + + it("closes the write server", async () => { + close({ project_id, compute_server_id }); + for (const f of cleanups) { + f(); + } + }); +}); + +describe("do a more challenging test that involves a larger file thathas to be broken into many chunks", () => { + const project_id = "00000000-0000-4000-8000-000000000000"; + const compute_server_id = 1; + + it("create the write server", async () => { + await createServer({ + project_id, + compute_server_id, + createWriteStream, + }); + }); + + let cleanups: any[] = []; + let CONTENT = ""; + for (let i = 0; i < 1000000; i++) { + CONTENT += `${i}`; + } + let source; + it("creates the file we will read", async () => { + const { path, cleanup } = await tempFile(); + source = path; + await fsWriteFile(path, CONTENT); + cleanups.push(cleanup); + }); + + let dest; + it("write to a new file", async () => { + const { path, cleanup } = await tempFile(); + dest = path; + cleanups.push(cleanup); + + const stream = createReadStream(source); + const { bytes, chunks } = await writeFile({ + stream, + project_id, + compute_server_id, + path, + }); + expect(chunks).toBeGreaterThan(1); + expect(bytes).toBe(CONTENT.length); + }); + + it("confirm that the dest file is correct", async () => { + const d = (await readFile(dest)).toString(); + expect(d.length).toEqual(CONTENT.length); + expect(d).toEqual(CONTENT); + }); + + it("closes the write server", async () => { + close({ project_id, compute_server_id }); + for (const f of cleanups) { + f(); + } + }); +}); diff --git a/src/packages/backend/nats/test/dko.test.ts b/src/packages/backend/nats/test/sync/dko.test.ts similarity index 100% rename from src/packages/backend/nats/test/dko.test.ts rename to src/packages/backend/nats/test/sync/dko.test.ts diff --git a/src/packages/backend/nats/test/dkv-merge.test.ts b/src/packages/backend/nats/test/sync/dkv-merge.test.ts similarity index 100% rename from src/packages/backend/nats/test/dkv-merge.test.ts rename to src/packages/backend/nats/test/sync/dkv-merge.test.ts diff --git a/src/packages/backend/nats/test/dkv.test.ts b/src/packages/backend/nats/test/sync/dkv.test.ts similarity index 100% rename from src/packages/backend/nats/test/dkv.test.ts rename to src/packages/backend/nats/test/sync/dkv.test.ts diff --git a/src/packages/backend/nats/test/dstream.test.ts b/src/packages/backend/nats/test/sync/dstream.test.ts similarity index 100% rename from src/packages/backend/nats/test/dstream.test.ts rename to src/packages/backend/nats/test/sync/dstream.test.ts diff --git a/src/packages/backend/nats/test/open-files.test.ts b/src/packages/backend/nats/test/sync/open-files.test.ts similarity index 100% rename from src/packages/backend/nats/test/open-files.test.ts rename to src/packages/backend/nats/test/sync/open-files.test.ts diff --git a/src/packages/backend/nats/test/time.test.ts b/src/packages/backend/nats/test/time.test.ts new file mode 100644 index 0000000000..d15fd3bf3f --- /dev/null +++ b/src/packages/backend/nats/test/time.test.ts @@ -0,0 +1,30 @@ +/* +DEVELOPMENT: + +pnpm exec jest --watch --forceExit --detectOpenHandles "time.test.ts" +*/ + +// this sets client +import "@cocalc/backend/nats"; + +import time, { getSkew } from "@cocalc/nats/time"; + +describe("get time from nats", () => { + it("tries to get the time before the skew, so it is not initialized yet", () => { + expect(time).toThrow("clock skew not known"); + }); + + it("gets the skew, so that time is initialized", async () => { + const skew = await getSkew(); + expect(Math.abs(skew)).toBeLessThan(1000); + }); + + it("gets the time, which should be close to our time on a test system", () => { + // times in ms, so divide by 1000 so expecting to be within a second + expect(time() / 1000).toBeCloseTo(Date.now() / 1000, 0); + }); + + it("time is a number", () => { + expect(typeof time()).toBe("number"); + }); +}); diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 991f41cdb5..75d44684ad 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -10,7 +10,10 @@ "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": ["utilities", "cocalc"], + "keywords": [ + "utilities", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -19,7 +22,12 @@ "test": "pnpm exec jest --forceExit --detectOpenHandles", "prepublishOnly": "pnpm test" }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { @@ -40,6 +48,7 @@ "rimraf": "^5.0.5", "shell-escape": "^0.2.0", "supports-color": "^9.0.2", + "tmp-promise": "^3.0.3", "underscore": "^1.12.1" }, "repository": { diff --git a/src/packages/nats/files/write.ts b/src/packages/nats/files/write.ts index 2e4c54496f..042b3d43ea 100644 --- a/src/packages/nats/files/write.ts +++ b/src/packages/nats/files/write.ts @@ -56,13 +56,15 @@ function getWriteSubject({ project_id, compute_server_id }) { }); } -let sub: Subscription | null = null; -export async function close() { - if (sub == null) { +let subs: { [name: string]: Subscription } = {}; +export async function close({ project_id, compute_server_id }) { + const key = getWriteSubject({ project_id, compute_server_id }); + if (subs[key] == null) { return; } + const sub = subs[key]; + delete subs[key]; await sub.drain(); - sub = null; } export async function createServer({ @@ -70,19 +72,23 @@ export async function createServer({ compute_server_id, createWriteStream, }) { + const subject = getWriteSubject({ project_id, compute_server_id }); + let sub = subs[subject]; if (sub != null) { return; } const { nc } = await getEnv(); - const subject = getWriteSubject({ project_id, compute_server_id }); sub = nc.subscribe(subject); - listen({ createWriteStream, project_id, compute_server_id }); + subs[subject] = sub; + listen({ sub, createWriteStream, project_id, compute_server_id }); } -async function listen({ createWriteStream, project_id, compute_server_id }) { - if (sub == null) { - return; - } +async function listen({ + sub, + createWriteStream, + project_id, + compute_server_id, +}) { // NOTE: we just handle as many messages as we get in parallel, so this // could be a large number of simultaneous downloads. These are all by // authenticated users of the project, and the load is on the project, @@ -99,8 +105,8 @@ async function handleMessage({ compute_server_id, }) { let error = ""; + const { jc } = await getEnv(); try { - const { jc } = await getEnv(); const { path, name, maxWait } = jc.decode(mesg.data); const writeStream = createWriteStream(path); writeStream.on("error", (err) => { @@ -108,6 +114,8 @@ async function handleMessage({ mesg.respond(jc.encode({ error, status: "error" })); console.warn(`error writing ${path}: ${error}`); }); + let chunks = 0; + let bytes = 0; for await (const chunk of await readFile({ project_id, compute_server_id, @@ -119,9 +127,11 @@ async function handleMessage({ return; } writeStream.write(chunk); + chunks += 1; + bytes += chunk.length; } writeStream.end(); - mesg.respond(jc.encode({ status: "success" })); + mesg.respond(jc.encode({ status: "success", bytes, chunks })); } catch (err) { if (!error) { mesg.respond(jc.encode({ error: `${err}`, status: "error" })); @@ -141,7 +151,7 @@ export async function writeFile({ path: string; stream: Readable; maxWait?: number; -}): Promise { +}): Promise<{ bytes: number; chunks: number }> { const name = randomId(); try { function createReadStream() { @@ -161,10 +171,11 @@ export async function writeFile({ jc.encode({ name, path, maxWait }), { timeout: maxWait }, ); - const { error } = jc.decode(resp.data); + const { error, bytes, chunks } = jc.decode(resp.data); if (error) { throw Error(error); } + return { bytes, chunks }; } finally { await closeReadService({ project_id, compute_server_id, name }); } diff --git a/src/packages/nats/time.ts b/src/packages/nats/time.ts index b781b6e3b7..e5f8994da8 100644 --- a/src/packages/nats/time.ts +++ b/src/packages/nats/time.ts @@ -63,7 +63,7 @@ async function syncLoop() { let dkv: any = null; const initDkv = reuseInFlight(async () => { const { account_id, project_id } = getClient(); - console.log({ account_id, project_id, client: getClient() }); + // console.log({ account_id, project_id, client: getClient() }); dkv = await createDkv({ account_id, project_id, diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index d21c0965dd..a0b15d6ce6 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -124,6 +124,9 @@ importers: supports-color: specifier: ^9.0.2 version: 9.4.0 + tmp-promise: + specifier: ^3.0.3 + version: 3.0.3 underscore: specifier: ^1.12.1 version: 1.13.7 @@ -12016,10 +12019,17 @@ packages: tinyqueue@3.0.0: resolution: {integrity: sha512-gRa9gwYU3ECmQYv3lslts5hxuIa90veaEcxDYuu3QGOIAEM2mOZkVHp48ANJuu1CURtRdHKUBY5Lm1tHV+sD4g==} + tmp-promise@3.0.3: + resolution: {integrity: sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==} + tmp@0.0.33: resolution: {integrity: sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==} engines: {node: '>=0.6.0'} + tmp@0.2.3: + resolution: {integrity: sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==} + engines: {node: '>=14.14'} + tmpl@1.0.5: resolution: {integrity: sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==} @@ -24731,10 +24741,16 @@ snapshots: tinyqueue@3.0.0: {} + tmp-promise@3.0.3: + dependencies: + tmp: 0.2.3 + tmp@0.0.33: dependencies: os-tmpdir: 1.0.2 + tmp@0.2.3: {} + tmpl@1.0.5: {} to-fast-properties@2.0.0: {} From 1e2a70406e2b0e577511ad81703227e79276d0de Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 21 Feb 2025 03:31:07 +0000 Subject: [PATCH 252/281] nats: write unit tests of streaming reads --- .../backend/nats/test/files/read.test.ts | 105 ++++++++++++++++++ .../backend/nats/test/files/write.test.ts | 10 +- src/packages/nats/files/read.ts | 2 +- 3 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 src/packages/backend/nats/test/files/read.test.ts diff --git a/src/packages/backend/nats/test/files/read.test.ts b/src/packages/backend/nats/test/files/read.test.ts new file mode 100644 index 0000000000..48bb2c860c --- /dev/null +++ b/src/packages/backend/nats/test/files/read.test.ts @@ -0,0 +1,105 @@ +/* +Test async streaming read of files from a compute servers using NATS. + + +DEVELOPMENT: + +pnpm exec jest --watch --forceExit --detectOpenHandles "read.test.ts" + +*/ + +import "@cocalc/backend/nats"; +import { close, createServer, readFile } from "@cocalc/nats/files/read"; +import { createReadStream } from "fs"; +import { file as tempFile } from "tmp-promise"; +import { writeFile as fsWriteFile } from "fs/promises"; +import { sha1 } from "@cocalc/backend/sha1"; + +describe("do a basic test that the file read service works", () => { + const project_id = "00000000-0000-4000-8000-000000000000"; + const compute_server_id = 0; + it("create the read server", async () => { + await createServer({ + project_id, + compute_server_id, + createReadStream, + }); + }); + + let cleanups: any[] = []; + const CONTENT = "cocalc"; + let source; + it("creates the file we will read", async () => { + const { path, cleanup } = await tempFile(); + source = path; + await fsWriteFile(path, CONTENT); + cleanups.push(cleanup); + }); + + it("reads the file into memory", async () => { + const r = await readFile({ project_id, compute_server_id, path: source }); + // will get just one chunk + for await (const chunk of r) { + expect(chunk.toString()).toEqual(CONTENT); + } + }); + + it("closes the write server", async () => { + close({ project_id, compute_server_id }); + for (const f of cleanups) { + f(); + } + }); +}); + +describe("do a larger test that involves multiple chunks and a different name", () => { + const project_id = "00000000-0000-4000-8000-000000000000"; + const compute_server_id = 0; + const name = "b"; + it("create the read server", async () => { + await createServer({ + project_id, + compute_server_id, + createReadStream, + name, + }); + }); + + let cleanups: any[] = []; + let CONTENT = ""; + for (let i = 0; i < 1000000; i++) { + CONTENT += `${i}`; + } + let source; + it("creates the file we will read", async () => { + const { path, cleanup } = await tempFile(); + source = path; + await fsWriteFile(path, CONTENT); + cleanups.push(cleanup); + }); + + it("reads the file into memory", async () => { + const r = await readFile({ + project_id, + compute_server_id, + path: source, + name, + }); + // will get many chunks. + let chunks: Buffer[] = []; + for await (const chunk of r) { + chunks.push(chunk); + } + expect(chunks.length).toBeGreaterThan(1); + const s = Buffer.concat(chunks).toString(); + expect(s.length).toBe(CONTENT.length); + expect(sha1(s)).toEqual(sha1(CONTENT)); + }); + + it("closes the write server", async () => { + close({ project_id, compute_server_id, name }); + for (const f of cleanups) { + f(); + } + }); +}); diff --git a/src/packages/backend/nats/test/files/write.test.ts b/src/packages/backend/nats/test/files/write.test.ts index c51a8ad0d3..f0711f8ac9 100644 --- a/src/packages/backend/nats/test/files/write.test.ts +++ b/src/packages/backend/nats/test/files/write.test.ts @@ -4,7 +4,8 @@ Test async streaming writing of files to compute servers using NATS. DEVELOPMENT: -pnpm exec jest --watch --forceExit --detectOpenHandles "write.test.ts" + pnpm exec jest --watch --forceExit --detectOpenHandles "write.test.ts" + */ import "@cocalc/backend/nats"; @@ -12,6 +13,7 @@ import { close, createServer, writeFile } from "@cocalc/nats/files/write"; import { createWriteStream, createReadStream } from "fs"; import { file as tempFile } from "tmp-promise"; import { writeFile as fsWriteFile, readFile } from "fs/promises"; +import { sha1 } from "@cocalc/backend/sha1"; describe("do a basic test that the file writing service works", () => { const project_id = "00000000-0000-4000-8000-000000000000"; @@ -64,7 +66,7 @@ describe("do a basic test that the file writing service works", () => { }); }); -describe("do a more challenging test that involves a larger file thathas to be broken into many chunks", () => { +describe("do a more challenging test that involves a larger file that has to be broken into many chunks", () => { const project_id = "00000000-0000-4000-8000-000000000000"; const compute_server_id = 1; @@ -109,7 +111,9 @@ describe("do a more challenging test that involves a larger file thathas to be b it("confirm that the dest file is correct", async () => { const d = (await readFile(dest)).toString(); expect(d.length).toEqual(CONTENT.length); - expect(d).toEqual(CONTENT); + // not directly comparing, since huge and if something goes wrong the output + // saying the test failed is huge. + expect(sha1(d)).toEqual(sha1(CONTENT)); }); it("closes the write server", async () => { diff --git a/src/packages/nats/files/read.ts b/src/packages/nats/files/read.ts index 966618afe6..c5a2a8bb96 100644 --- a/src/packages/nats/files/read.ts +++ b/src/packages/nats/files/read.ts @@ -154,7 +154,7 @@ export async function* readFile({ bytes = resp.data.length; // console.log("received seq", { seq: next, bytes }); if (next != seq + 1) { - throw Error("lost data"); + throw Error(`lost data: seq=${seq}, next=${next}`); } seq = next; } From 7917aee51918e0793d60433156bffb16158128ef Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 21 Feb 2025 03:40:13 +0000 Subject: [PATCH 253/281] nats: fix some unit tests and add some more docs --- .../backend/nats/test/sync/open-files.test.ts | 28 +------------------ src/packages/nats/files/read.ts | 2 ++ src/packages/nats/files/write.ts | 28 ++++++++++++++++++- src/packages/nats/time.ts | 5 ++++ 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/src/packages/backend/nats/test/sync/open-files.test.ts b/src/packages/backend/nats/test/sync/open-files.test.ts index 9cc1e069dd..12a00a6af3 100644 --- a/src/packages/backend/nats/test/sync/open-files.test.ts +++ b/src/packages/backend/nats/test/sync/open-files.test.ts @@ -52,12 +52,7 @@ describe("create open file tracker and do some basic operations", () => { it("touch file in one and observe change and timestamp getting assigned by server", async () => { o1.touch(file1); - expect(o1.get(file1)?.time).toBe(undefined); - o1.save(); - if (o1.get(file1)?.time == null) { - await once(o1, "change", 250); - expect(o1.get(file1).path).toBe(file1); - } + expect(o1.get(file1).time).toBeCloseTo(Date.now(), -3); }); it("touches file in one and observes change by OTHER", async () => { @@ -93,27 +88,6 @@ describe("create open file tracker and do some basic operations", () => { expect(o2.getAll().length).toBe(1); }); - it("closes file2", async () => { - expect(o2.get(file2).open).toBe(true); - o2.closeFile(file2); - expect(o2.get(file2).open).toBe(false); - o2.save(); - if (o1.get(file2).open) { - await once(o1, "change", 250); - } - expect(o1.get(file2).open).toBe(false); - }); - - it("touching a closed file re-opens it", async () => { - o2.touch(file2); - expect(o2.get(file2).open).toBe(true); - o2.save(); - if (!o1.get(file2).open) { - await once(o1, "change", 250); - } - expect(o1.get(file2).open).toBe(true); - }); - it("sets an error", async () => { o2.setError(file2, Error("test error")); expect(o2.get(file2).error.error).toBe("Error: test error"); diff --git a/src/packages/nats/files/read.ts b/src/packages/nats/files/read.ts index c5a2a8bb96..4d226cfc96 100644 --- a/src/packages/nats/files/read.ts +++ b/src/packages/nats/files/read.ts @@ -24,6 +24,8 @@ over a websocket for compute servers, so would just copy that code. DEVELOPMENT: +See src/packages/backend/nats/test/files/read.test.ts for unit tests. + ~/cocalc/src/packages/backend$ node require('@cocalc/backend/nats'); a = require('@cocalc/nats/files/read'); a.createServer({project_id:'00847397-d6a8-4cb0-96a8-6ef64ac3e6cf',compute_server_id:0,createReadStream:require('fs').createReadStream}) diff --git a/src/packages/nats/files/write.ts b/src/packages/nats/files/write.ts index 042b3d43ea..0ee189b9b2 100644 --- a/src/packages/nats/files/write.ts +++ b/src/packages/nats/files/write.ts @@ -2,7 +2,31 @@ Streaming write over NATS to a project or compute server. This is a key component to support user uploads, while being memory efficient -by streaming the write. +by streaming the write. Basically it uses NATS to support efficiently doing +streaming writes of files to any compute server or project that is somehow +connected to NATS. + +INSTRUCTIONS: + +Import writeFile: + + import { writeFile } from "@cocalc/nats/files/write"; + +Now you can write a given path to a project (or compute_server) as +simply as this: + + const stream = createReadStream('a file') + await writeFile({stream, project_id, compute_server_id, path, maxWait}) + +- Here stream can be any readable stream, not necessarily a stream made using + a file. E.g., you could use PassThrough and explicitly write to it by + write calls. + +- maxWait is a time in ms after which if the file isn't fully written, everything + is cleaned up and there is an error. + + +HOW THIS WORKS: Here's how this works from the side of the compute server: @@ -26,6 +50,8 @@ Here's how it works from the side of whoever is sending the file: DEVELOPMENT: +See src/packages/backend/nats/test/files/write.test.ts for unit tests. + ~/cocalc/src/packages/backend$ node require('@cocalc/backend/nats'); a = require('@cocalc/nats/files/write'); diff --git a/src/packages/nats/time.ts b/src/packages/nats/time.ts index e5f8994da8..686e852e99 100644 --- a/src/packages/nats/time.ts +++ b/src/packages/nats/time.ts @@ -21,6 +21,11 @@ getTime(); // -- ms since the epoch // once this works you can definitely call getTime henceforth. await getSkew(); +DEVELOPMENT: + +See src/packages/backend/nats/test/time.test.ts for unit tests. + + */ import { dkv as createDkv } from "@cocalc/nats/sync/dkv"; From 2bc2c081bbbe8c3b39456830e940135c2dc359f6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 21 Feb 2025 13:14:11 +0000 Subject: [PATCH 254/281] nats file upload: starting work integrating this with hub (work in progress) --- src/packages/frontend/file-upload.tsx | 24 +- src/packages/hub/package.json | 2 +- src/packages/hub/servers/app/upload.ts | 92 ++ src/packages/hub/servers/express-app.ts | 2 + src/packages/pnpm-lock.yaml | 1180 ++++++++++++----------- src/packages/project/package.json | 3 +- 6 files changed, 719 insertions(+), 584 deletions(-) create mode 100644 src/packages/hub/servers/app/upload.ts diff --git a/src/packages/frontend/file-upload.tsx b/src/packages/frontend/file-upload.tsx index e737631d4e..c2e2008ffa 100644 --- a/src/packages/frontend/file-upload.tsx +++ b/src/packages/frontend/file-upload.tsx @@ -25,7 +25,7 @@ import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import { labels } from "@cocalc/frontend/i18n"; import { BASE_URL } from "@cocalc/frontend/misc"; import { MAX_BLOB_SIZE } from "@cocalc/util/db-schema/blobs"; -import { defaults, encode_path, is_array, merge } from "@cocalc/util/misc"; +import { defaults, is_array, merge } from "@cocalc/util/misc"; // 3GB upload limit -- since that's the default filesystem quota // and it should be plenty? @@ -103,19 +103,27 @@ function Header({ close_preview }: { close_preview?: Function }) { ); } -function postUrl(project_id: string, path: string): string { +function postUrl( + project_id: string, + path: string, + compute_server_id?: number, +): string { if (!project_id) { return join(appBasePath, "blobs"); } - const dest_dir = encode_path(path); - const compute_server_id = redux - .getProjectStore(project_id) - .get("compute_server_id"); + if (compute_server_id == null) { + compute_server_id = + redux.getProjectStore(project_id).get("compute_server_id") ?? 0; + } return join( appBasePath, - project_id, - `raw/.smc/upload?dest_dir=${dest_dir}&compute_server_id=${compute_server_id}`, + `upload?project_id=${project_id}&compute_server_id=${compute_server_id}&path=${encodeURIComponent(path)}`, ); + // return join( + // appBasePath, + // project_id, + // `raw/.smc/upload?dest_dir=${dest_dir}&compute_server_id=${compute_server_id}`, + // ); } interface FileUploadProps { diff --git a/src/packages/hub/package.json b/src/packages/hub/package.json index 902d750289..6d6e0c3c4e 100644 --- a/src/packages/hub/package.json +++ b/src/packages/hub/package.json @@ -38,7 +38,7 @@ "debug": "^4.4.0", "escape-html": "^1.0.3", "express": "^4.21.2", - "formidable": "^3.5.1", + "formidable": "^3.5.2", "http-proxy": "^1.18.1", "immutable": "^4.3.0", "jquery": "^3.6.0", diff --git a/src/packages/hub/servers/app/upload.ts b/src/packages/hub/servers/app/upload.ts new file mode 100644 index 0000000000..764f9c9861 --- /dev/null +++ b/src/packages/hub/servers/app/upload.ts @@ -0,0 +1,92 @@ +/* +Support user uploading files directly to CoCalc from their browsers. + +- uploading to projects and compute servers, with full support for potentially + very LARGE file uploads that stream via NATS. This checks users is authenticated + with write access. + +- uploading blobs to our database. + +Which of the above happens depends on query params. + +NOTE: Code for downloading files from projects/compute servers +is in the middle of packages/hub/proxy/handle-request.ts +*/ + +// See also ./blob-upload.ts, which is similar, but targets our main +// database instead of projects, and doesn't need to worry about streaming. + +import { Router } from "express"; +import { getLogger } from "@cocalc/hub/logger"; +import getAccount from "@cocalc/server/auth/get-account"; +import isCollaborator from "@cocalc/server/projects/is-collaborator"; +import formidable from "formidable"; +import { readFile, unlink } from "fs/promises"; + +const logger = getLogger("hub:servers:app:blob-upload"); +function dbg(...args): void { + logger.debug("upload ", ...args); +} + +export default function init(router: Router) { + router.post("/upload", async (req, res) => { + const account_id = await getAccount(req); + if (!account_id) { + res.status(500).send("user must be signed in to upload files"); + return; + } + const { project_id, compute_server_id, dest, ttl, blob } = req.query; + if (!blob || project_id) { + if ( + typeof project_id != "string" || + !(await isCollaborator({ account_id, project_id })) + ) { + res.status(500).send("user must be collaborator on project"); + return; + } + } + + dbg({ account_id, project_id, compute_server_id, dest, ttl, blob }); + + try { + const form = formidable({ + keepExtensions: true, + hashAlgorithm: "sha1", + }); + + dbg("parsing form data..."); + // https://github.com/node-formidable/formidable?tab=readme-ov-file#parserequest-callback + const [_, files] = await form.parse(req); + //dbg(`finished parsing form data. ${JSON.stringify({ fields, files })}`); + + /* Just for the sake of understanding this, this is how this looks like in the real world (formidable@3): + > files.file[0] + { + size: 80789, + filepath: '/home/hsy/p/cocalc/src/data/projects/c8787b71-a85f-437b-9d1b-29833c3a199e/asdf/asdf/8e3e4367333e45275a8d1aa03.png', + newFilename: '8e3e4367333e45275a8d1aa03.png', + mimetype: 'application/octet-stream', + mtime: '2024-04-23T09:25:53.197Z', + originalFilename: 'Screenshot from 2024-04-23 09-20-40.png' + } + */ + if (files.file?.[0] != null) { + const { filepath } = files.file[0]; + try { + dbg("got", files); + dbg("got ", await readFile(filepath)); + } finally { + try { + await unlink(filepath); + } catch (err) { + dbg("WARNING -- failed to delete uploaded file", err); + } + } + } + res.send({ status: "ok", files }); + } catch (err) { + dbg("upload failed ", err); + res.status(500).send(`upload failed -- ${err}`); + } + }); +} diff --git a/src/packages/hub/servers/express-app.ts b/src/packages/hub/servers/express-app.ts index c632987730..d2776f339d 100644 --- a/src/packages/hub/servers/express-app.ts +++ b/src/packages/hub/servers/express-app.ts @@ -22,6 +22,7 @@ import initProxy from "../proxy"; import initAPI from "./app/api"; import initAppRedirect from "./app/app-redirect"; import initBlobUpload from "./app/blob-upload"; +import initUpload from "./app/upload"; import initBlobs from "./app/blobs"; import initCustomize from "./app/customize"; import { initMetricsEndpoint, setupInstrumentation } from "./app/metrics"; @@ -126,6 +127,7 @@ export default async function init(opts: Options): Promise<{ initBlobs(router); initBlobUpload(router); + initUpload(router); initSetCookies(router); initNatsServer(router); initCustomize(router, opts.isPersonal); diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index a0b15d6ce6..62cb156a71 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -62,7 +62,7 @@ importers: version: 3.7.1 url-loader: specifier: ^4.1.1 - version: 4.1.1(webpack@5.97.1(uglify-js@3.19.3)) + version: 4.1.1(webpack@5.98.0(uglify-js@3.19.3)) devDependencies: '@cocalc/backend': specifier: workspace:* @@ -354,7 +354,7 @@ importers: version: 4.1.12 '@uiw/react-textarea-code-editor': specifier: ^2.1.1 - version: 2.1.9(@babel/runtime@7.26.7)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + version: 2.1.9(@babel/runtime@7.26.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@use-gesture/react': specifier: ^10.2.24 version: 10.3.1(react@18.3.1) @@ -567,7 +567,7 @@ importers: version: 7.1.1 plotly.js: specifier: ^2.29.1 - version: 2.35.2(@rspack/core@1.2.2(@swc/helpers@0.5.15))(mapbox-gl@3.9.4)(webpack@5.97.1) + version: 2.35.2(@rspack/core@1.2.5(@swc/helpers@0.5.15))(mapbox-gl@3.10.0)(webpack@5.98.0) project-name-generator: specifier: ^2.1.6 version: 2.1.9 @@ -609,10 +609,10 @@ importers: version: 7.1.0(react@18.3.1)(typescript@5.7.3) react-plotly.js: specifier: ^2.6.0 - version: 2.6.0(plotly.js@2.35.2(@rspack/core@1.2.2(@swc/helpers@0.5.15))(mapbox-gl@3.9.4)(webpack@5.97.1))(react@18.3.1) + version: 2.6.0(plotly.js@2.35.2(@rspack/core@1.2.5(@swc/helpers@0.5.15))(mapbox-gl@3.10.0)(webpack@5.98.0))(react@18.3.1) react-redux: specifier: ^8.0.5 - version: 8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@4.2.1) + version: 8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@5.0.1) react-timeago: specifier: ^7.2.0 version: 7.2.0(react@18.3.1) @@ -834,8 +834,8 @@ importers: specifier: ^4.21.2 version: 4.21.2 formidable: - specifier: ^3.5.1 - version: 3.5.1 + specifier: ^3.5.2 + version: 3.5.2 http-proxy: specifier: ^1.18.1 version: 1.18.1(debug@4.4.0) @@ -871,7 +871,7 @@ importers: version: 2.29.1 next: specifier: 14.2.22 - version: 14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.83.4) + version: 14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.0) nyc: specifier: ^15.1.0 version: 15.1.0 @@ -931,7 +931,7 @@ importers: version: 13.12.0 webpack-dev-middleware: specifier: ^7.4.2 - version: 7.4.2(webpack@5.97.1(uglify-js@3.19.3)) + version: 7.4.2(webpack@5.98.0(uglify-js@3.19.3)) webpack-hot-middleware: specifier: ^2.26.1 version: 2.26.1 @@ -1195,16 +1195,16 @@ importers: version: 2.1.2 next: specifier: 14.2.22 - version: 14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.83.4) + version: 14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.0) next-remove-imports: specifier: ^1.0.11 - version: 1.0.12(webpack@5.97.1) + version: 1.0.12(webpack@5.98.0) next-rest-framework: specifier: 6.0.0-beta.4 version: 6.0.0-beta.4(zod@3.23.8) next-translate: specifier: ^2.6.2 - version: 2.6.2(next@14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.83.4))(react@18.3.1) + version: 2.6.2(next@14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.0))(react@18.3.1) password-hash: specifier: ^1.2.2 version: 1.2.2 @@ -1320,6 +1320,9 @@ importers: '@nteract/messaging': specifier: ^7.0.20 version: 7.0.20 + '@types/formidable': + specifier: ^3.4.5 + version: 3.4.5 '@types/lodash': specifier: ^4.14.202 version: 4.17.9 @@ -1491,7 +1494,7 @@ importers: version: 0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) '@langchain/community': specifier: ^0.3.24 - version: 0.3.24(@browserbasehq/sdk@2.0.0(encoding@0.1.13))(@browserbasehq/stagehand@1.10.1(@playwright/test@1.50.0)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.3.2(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@8.7.0)(cheerio@1.0.0-rc.10)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.1)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.1)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.1)(playwright@1.50.0)(ws@8.18.0) + version: 0.3.24(@browserbasehq/sdk@2.3.0(encoding@0.1.13))(@browserbasehq/stagehand@1.13.0(@playwright/test@1.50.1)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.5.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@11.8.1)(cheerio@1.0.0)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.3)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.3)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.3)(playwright@1.50.1)(ws@8.18.1) '@langchain/core': specifier: ^0.3.30 version: 0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) @@ -1739,7 +1742,7 @@ importers: devDependencies: '@rspack/cli': specifier: ^1.1.1 - version: 1.1.1(@rspack/core@1.1.1(@swc/helpers@0.5.15))(@types/express@4.17.21)(webpack@5.97.1) + version: 1.1.1(@rspack/core@1.1.1(@swc/helpers@0.5.15))(@types/express@4.17.21)(webpack@5.98.0) '@rspack/core': specifier: ^1.1.1 version: 1.1.1(@swc/helpers@0.5.15) @@ -1787,19 +1790,19 @@ importers: version: 3.0.0 clean-webpack-plugin: specifier: ^4.0.0 - version: 4.0.0(webpack@5.97.1) + version: 4.0.0(webpack@5.98.0) coffee-cache: specifier: ^1.0.2 version: 1.0.2 coffee-loader: specifier: ^3.0.0 - version: 3.0.0(coffeescript@2.7.0)(webpack@5.97.1) + version: 3.0.0(coffeescript@2.7.0)(webpack@5.98.0) coffeescript: specifier: ^2.5.1 version: 2.7.0 css-loader: specifier: ^7.1.2 - version: 7.1.2(@rspack/core@1.1.1(@swc/helpers@0.5.15))(webpack@5.97.1) + version: 7.1.2(@rspack/core@1.1.1(@swc/helpers@0.5.15))(webpack@5.98.0) entities: specifier: ^2.2.0 version: 2.2.0 @@ -1811,7 +1814,7 @@ importers: version: 9.1.0(eslint@8.57.1) eslint-plugin-prettier: specifier: ^5.1.3 - version: 5.2.1(@types/eslint@9.6.1)(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.4.2) + version: 5.2.1(@types/eslint@9.6.1)(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.5.1) eslint-plugin-react: specifier: ^7.33.2 version: 7.36.1(eslint@8.57.1) @@ -1826,16 +1829,16 @@ importers: version: 1.7.3(handlebars@4.7.8) html-loader: specifier: ^2.1.2 - version: 2.1.2(webpack@5.97.1) + version: 2.1.2(webpack@5.98.0) html-webpack-plugin: specifier: ^5.5.3 - version: 5.6.0(@rspack/core@1.1.1(@swc/helpers@0.5.15))(webpack@5.97.1) + version: 5.6.0(@rspack/core@1.1.1(@swc/helpers@0.5.15))(webpack@5.98.0) identity-obj-proxy: specifier: ^3.0.0 version: 3.0.0 imports-loader: specifier: ^3.0.0 - version: 3.1.1(webpack@5.97.1) + version: 3.1.1(webpack@5.98.0) jquery: specifier: ^3.6.0 version: 3.7.1 @@ -1859,13 +1862,13 @@ importers: version: 4.2.0 less-loader: specifier: ^11.0.0 - version: 11.1.4(less@4.2.0)(webpack@5.97.1) + version: 11.1.4(less@4.2.0)(webpack@5.98.0) path-browserify: specifier: ^1.0.1 version: 1.0.1 raw-loader: specifier: ^4.0.2 - version: 4.0.2(webpack@5.97.1) + version: 4.0.2(webpack@5.98.0) react: specifier: ^18.3.1 version: 18.3.1 @@ -1886,25 +1889,25 @@ importers: version: 1.79.3 sass-loader: specifier: ^16.0.2 - version: 16.0.2(@rspack/core@1.1.1(@swc/helpers@0.5.15))(sass@1.79.3)(webpack@5.97.1) + version: 16.0.2(@rspack/core@1.1.1(@swc/helpers@0.5.15))(sass@1.79.3)(webpack@5.98.0) script-loader: specifier: ^0.7.2 version: 0.7.2 source-map-loader: specifier: ^3.0.0 - version: 3.0.2(webpack@5.97.1) + version: 3.0.2(webpack@5.98.0) stream-browserify: specifier: ^3.0.0 version: 3.0.0 style-loader: specifier: ^2.0.0 - version: 2.0.0(webpack@5.97.1) + version: 2.0.0(webpack@5.98.0) timeago: specifier: ^1.6.3 version: 1.6.7 ts-jest: specifier: ^29.2.3 - version: 29.2.5(@babel/core@7.26.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(jest@29.7.0(@types/node@18.19.50))(typescript@5.7.3) + version: 29.2.5(@babel/core@7.26.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.9))(jest@29.7.0(@types/node@18.19.50))(typescript@5.7.3) tsd: specifier: ^0.22.0 version: 0.22.0 @@ -1962,7 +1965,7 @@ importers: version: 18.19.50 ts-jest: specifier: ^29.2.3 - version: 29.2.5(@babel/core@7.26.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(jest@29.7.0(@types/node@18.19.50))(typescript@5.7.3) + version: 29.2.5(@babel/core@7.26.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.9))(jest@29.7.0(@types/node@18.19.50))(typescript@5.7.3) sync-client: dependencies: @@ -2323,8 +2326,8 @@ packages: resolution: {integrity: sha512-ZsysZyXY4Tlx+Q53XdnOFmqwfB9QDTHYxaZYajWRoBLuLEAwI2UIbtxOjWh/cFaa9IKUlcB+DDuoskLuKu56JA==} engines: {node: '>=6.9.0'} - '@babel/compat-data@7.26.5': - resolution: {integrity: sha512-XvcZi1KWf88RVbF9wn8MN6tYFloU5qX8KjuF3E1PVBmJ9eypXfs4GRiJwLuTZL0iSnJUKn1BFPa5BPZZJyFzPg==} + '@babel/compat-data@7.26.8': + resolution: {integrity: sha512-oH5UPLMWR3L2wEFLnFJ1TZXqHufiTKAiLfqw5zkhS4dKXLJ10yVztfil/twG8EDTA4F/tvVNw9nOl4ZMslB8rQ==} engines: {node: '>=6.9.0'} '@babel/core@7.25.2': @@ -2335,8 +2338,8 @@ packages: resolution: {integrity: sha512-Oixnb+DzmRT30qu9d3tJSQkxuygWm32DFykT4bRoORPa9hZ/L4KhVB/XiRm6KG+roIEM7DBQlmg27kw2HZkdZg==} engines: {node: '>=6.9.0'} - '@babel/core@7.26.7': - resolution: {integrity: sha512-SRijHmF0PSPgLIBYlWnG0hyeJLwXE2CgpsXaMOrtt2yp9/86ALw6oUlj9KYuZ0JN07T4eBMVIW4li/9S1j2BGA==} + '@babel/core@7.26.9': + resolution: {integrity: sha512-lWBYIrF7qK5+GjY5Uy+/hEgp8OJWOD/rpy74GplYRhEauvbHDeFB8t5hPOZxCZ0Oxf4Cc36tK51/l3ymJysrKw==} engines: {node: '>=6.9.0'} '@babel/generator@7.25.6': @@ -2347,8 +2350,8 @@ packages: resolution: {integrity: sha512-5Dqpl5fyV9pIAD62yK9P7fcA768uVPUyrQmqpqstHWgMma4feF1x/oFysBCVZLY5wJ2GkMUCdsNDnGZrPoR6rA==} engines: {node: '>=6.9.0'} - '@babel/generator@7.26.5': - resolution: {integrity: sha512-2caSP6fN9I7HOe6nqhtft7V4g7/V/gfDsC3Ag4W7kEzzvRGKqiv0pu0HogPiZ3KaVSoNDhUws6IJjDjpfmYIXw==} + '@babel/generator@7.26.9': + resolution: {integrity: sha512-kEWdzjOAUMW4hAyrzJ0ZaTOu9OmpyDIQicIh0zg0EEcEkYXZb2TjtBhnHi2ViX7PKwZqF4xwqfAm299/QMP3lg==} engines: {node: '>=6.9.0'} '@babel/helper-annotate-as-pure@7.24.7': @@ -2477,8 +2480,8 @@ packages: resolution: {integrity: sha512-Sv6pASx7Esm38KQpF/U/OXLwPPrdGHNKoeblRxgZRLXnAtnkEe4ptJPDtAZM7fBLadbc1Q07kQpSiGQ0Jg6tRA==} engines: {node: '>=6.9.0'} - '@babel/helpers@7.26.7': - resolution: {integrity: sha512-8NHiL98vsi0mbPQmYAGWwfcFaOy4j2HY49fXJCfuDcdE7fMIsH9a7GdaeXpIBsbT7307WU8KCMp5pUVDNL4f9A==} + '@babel/helpers@7.26.9': + resolution: {integrity: sha512-Mz/4+y8udxBKdmzt/UjPACs4G3j5SshJJEFFKxlCGPydG4JAHXxjWjAwjd09tf6oINvl1VfMJo+nB7H2YKQ0dA==} engines: {node: '>=6.9.0'} '@babel/highlight@7.24.7': @@ -2508,8 +2511,8 @@ packages: engines: {node: '>=6.0.0'} hasBin: true - '@babel/parser@7.26.7': - resolution: {integrity: sha512-kEvgGGgEjRUutvdVvZhbn/BxVt+5VSpwXz1j3WYXQbXDo8KzFOPNG2GQbdAiNq8g6wn1yKk7C/qrke03a84V+w==} + '@babel/parser@7.26.9': + resolution: {integrity: sha512-81NWa1njQblgZbQHxWHpxxCzNsa3ZwvFqpUg7P+NNUU6f3UU2jBEg4OlF/J6rl8+PQGh1q6/zWScd001YwcA5A==} engines: {node: '>=6.0.0'} hasBin: true @@ -2616,8 +2619,8 @@ packages: resolution: {integrity: sha512-FjoyLe754PMiYsFaN5C94ttGiOmBNYTf6pLr4xXHAT5uctHb092PBszndLDR5XA/jghQvn4n7JMHl7dmTgbm9w==} engines: {node: '>=6.9.0'} - '@babel/runtime@7.26.7': - resolution: {integrity: sha512-AOPI3D+a8dXnja+iwsUqGRjr1BbZIe771sXdapOtYI531gSqpi92vXivKcq2asu/DFpdl1ceFAKZyRzK2PCVcQ==} + '@babel/runtime@7.26.9': + resolution: {integrity: sha512-aA63XwOkcl4xxQa3HjPMqOP6LiK0ZDv3mUPYEFXkpHbaFjtGggE1A61FjFzJnB+p7/oy2gA8E+rcBNl/zC1tMg==} engines: {node: '>=6.9.0'} '@babel/template@7.25.0': @@ -2628,6 +2631,10 @@ packages: resolution: {integrity: sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==} engines: {node: '>=6.9.0'} + '@babel/template@7.26.9': + resolution: {integrity: sha512-qyRplbeIpNZhmzOysF/wFMuP9sctmh2cFzRAZOn1YapxBsE1i9bJIY586R/WBLfLcmcBlM8ROBiQURnnNy+zfA==} + engines: {node: '>=6.9.0'} + '@babel/traverse@7.25.6': resolution: {integrity: sha512-9Vrcx5ZW6UwK5tvqsj0nGpp/XzqthkT0dqIc9g1AdtygFToNtTF67XzYS//dm+SAK9cp3B9R4ZO/46p63SCjlQ==} engines: {node: '>=6.9.0'} @@ -2636,8 +2643,8 @@ packages: resolution: {integrity: sha512-jatJPT1Zjqvh/1FyJs6qAHL+Dzb7sTb+xr7Q+gM1b+1oBsMsQQ4FkVKb6dFlJvLlVssqkRzV05Jzervt9yhnzg==} engines: {node: '>=6.9.0'} - '@babel/traverse@7.26.7': - resolution: {integrity: sha512-1x1sgeyRLC3r5fQOM0/xtQKsYjyxmFjaOrLJNtZ81inNjyJHGIolTULPiSc/2qe1/qfpFLisLQYFnnZl7QoedA==} + '@babel/traverse@7.26.9': + resolution: {integrity: sha512-ZYW7L+pL8ahU5fXmNbPF+iZFHCv5scFak7MZ9bwaRPLUhHh7QQEMjZUg0HevihoqCM5iSYHN61EyCoZvqC+bxg==} engines: {node: '>=6.9.0'} '@babel/types@7.25.6': @@ -2652,8 +2659,8 @@ packages: resolution: {integrity: sha512-OwS2CM5KocvQ/k7dFJa8i5bNGJP0hXWfVCfDkqRFP1IreH1JDC7wG6eCYCi0+McbfT8OR/kNqsI0UU0xP9H6PQ==} engines: {node: '>=6.9.0'} - '@babel/types@7.26.7': - resolution: {integrity: sha512-t8kDRGrKXyp6+tjUh7hw2RLyclsW4TRoRvRHtSyAX9Bb5ldlFh+90YAYY6awRXrlB4G5G2izNeGySpATlFzmOg==} + '@babel/types@7.26.9': + resolution: {integrity: sha512-Y3IR1cRnOxOCDvMmNiym7XpXQ93iGDDPHx+Zj+NM+rg0fBaShfQLkg+hKPaZCEvg5N/LeCo4+Rj/i3FuJsIQaw==} engines: {node: '>=6.9.0'} '@bcoe/v8-coverage@0.2.3': @@ -2662,11 +2669,11 @@ packages: '@braintree/sanitize-url@7.1.0': resolution: {integrity: sha512-o+UlMLt49RvtCASlOMW0AkHnabN9wR9rwCCherxO0yG4Npy34GkvrAqdXQvrhNs+jh+gkK8gB8Lf05qL/O7KWg==} - '@browserbasehq/sdk@2.0.0': - resolution: {integrity: sha512-BdPlZyn0dpXlL70gNK4acpqWIRB+edo2z0/GalQdWghRq8iQjySd9fVIF3evKH1p2wCYekZJRK6tm29YfXB67g==} + '@browserbasehq/sdk@2.3.0': + resolution: {integrity: sha512-H2nu46C6ydWgHY+7yqaP8qpfRJMJFVGxVIgsuHe1cx9HkfJHqzkuIqaK/k8mU4ZeavQgV5ZrJa0UX6MDGYiT4w==} - '@browserbasehq/stagehand@1.10.1': - resolution: {integrity: sha512-A222TCseFvKNvBwav7ZrZmug0JnYvy1vFI1ReNOtcymjhrZQLfklq1gm/luUjr8aRTbTzsUV8iclt5r0kyaXbA==} + '@browserbasehq/stagehand@1.13.0': + resolution: {integrity: sha512-7yyRUkbMUgV+8UOeHCDojpNHraK/ID8scOrCR1PG3tasuUrsieA+fA38YCVodQVsJSyiW0FBSLOYl7/23qYtkg==} peerDependencies: '@playwright/test': ^1.42.1 deepmerge: ^4.3.1 @@ -2968,9 +2975,6 @@ packages: peerDependencies: react: '>=16.8.0' - '@emnapi/runtime@1.3.1': - resolution: {integrity: sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==} - '@emotion/hash@0.8.0': resolution: {integrity: sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow==} @@ -3123,8 +3127,8 @@ packages: resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==} deprecated: Use @eslint/object-schema instead - '@ibm-cloud/watsonx-ai@1.3.2': - resolution: {integrity: sha512-I8FX086BG1dOOE+kRpsa28mDTTgc4qodjZsxWomKm6xfmgZFibC8WiJMjLx58QTLr6KZKw1Zk4ya7JVtturYag==} + '@ibm-cloud/watsonx-ai@1.5.0': + resolution: {integrity: sha512-jhZrpktR27xTHnCRw4agWsg1HnaIEvdrE1+3t40BzLCjqVgoLEdZh4Rw7/i6U9R/31VjqsD/UZMohNK21vBJmw==} engines: {node: '>=18.0.0'} '@iconify/types@2.0.0': @@ -3138,111 +3142,6 @@ packages: peerDependencies: react: '*' - '@img/sharp-darwin-arm64@0.33.5': - resolution: {integrity: sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [darwin] - - '@img/sharp-darwin-x64@0.33.5': - resolution: {integrity: sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [darwin] - - '@img/sharp-libvips-darwin-arm64@1.0.4': - resolution: {integrity: sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==} - cpu: [arm64] - os: [darwin] - - '@img/sharp-libvips-darwin-x64@1.0.4': - resolution: {integrity: sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==} - cpu: [x64] - os: [darwin] - - '@img/sharp-libvips-linux-arm64@1.0.4': - resolution: {integrity: sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==} - cpu: [arm64] - os: [linux] - - '@img/sharp-libvips-linux-arm@1.0.5': - resolution: {integrity: sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==} - cpu: [arm] - os: [linux] - - '@img/sharp-libvips-linux-s390x@1.0.4': - resolution: {integrity: sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==} - cpu: [s390x] - os: [linux] - - '@img/sharp-libvips-linux-x64@1.0.4': - resolution: {integrity: sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==} - cpu: [x64] - os: [linux] - - '@img/sharp-libvips-linuxmusl-arm64@1.0.4': - resolution: {integrity: sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==} - cpu: [arm64] - os: [linux] - - '@img/sharp-libvips-linuxmusl-x64@1.0.4': - resolution: {integrity: sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==} - cpu: [x64] - os: [linux] - - '@img/sharp-linux-arm64@0.33.5': - resolution: {integrity: sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [linux] - - '@img/sharp-linux-arm@0.33.5': - resolution: {integrity: sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm] - os: [linux] - - '@img/sharp-linux-s390x@0.33.5': - resolution: {integrity: sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [s390x] - os: [linux] - - '@img/sharp-linux-x64@0.33.5': - resolution: {integrity: sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [linux] - - '@img/sharp-linuxmusl-arm64@0.33.5': - resolution: {integrity: sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [linux] - - '@img/sharp-linuxmusl-x64@0.33.5': - resolution: {integrity: sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [linux] - - '@img/sharp-wasm32@0.33.5': - resolution: {integrity: sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [wasm32] - - '@img/sharp-win32-ia32@0.33.5': - resolution: {integrity: sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [ia32] - os: [win32] - - '@img/sharp-win32-x64@0.33.5': - resolution: {integrity: sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [win32] - '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -4263,8 +4162,8 @@ packages: resolution: {integrity: sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} - '@playwright/test@1.50.0': - resolution: {integrity: sha512-ZGNXbt+d65EGjBORQHuYKj+XhCewlwpnSd/EDuLPZGSiEWmgOJB5RmMCCYGy5aMfTs9wx61RivfDKi8H/hcMvw==} + '@playwright/test@1.50.1': + resolution: {integrity: sha512-Jii3aBg+CEDpgnuDxEp/h7BimHcUTDlpEtce89xEumlJ5ef2hqepZ+PWp1DDpYC/VO9fmWVI1IlEaoI5fK9FXQ==} engines: {node: '>=18'} hasBin: true @@ -4403,8 +4302,8 @@ packages: cpu: [arm64] os: [darwin] - '@rspack/binding-darwin-arm64@1.2.2': - resolution: {integrity: sha512-h23F8zEkXWhwMeScm0ZnN78Zh7hCDalxIWsm7bBS0eKadnlegUDwwCF8WE+8NjWr7bRzv0p3QBWlS5ufkcL4eA==} + '@rspack/binding-darwin-arm64@1.2.5': + resolution: {integrity: sha512-ou0NXMLp6RxY9Bx8P9lA8ArVjz/WAI/gSu5kKrdKKtMs6WKutl4vvP9A4HHZnISd9Tn00dlvDwNeNSUR7fjoDQ==} cpu: [arm64] os: [darwin] @@ -4413,8 +4312,8 @@ packages: cpu: [x64] os: [darwin] - '@rspack/binding-darwin-x64@1.2.2': - resolution: {integrity: sha512-vG5s7FkEvwrGLfksyDRHwKAHUkhZt1zHZZXJQn4gZKjTBonje8ezdc7IFlDiWpC4S+oBYp73nDWkUzkGRbSdcQ==} + '@rspack/binding-darwin-x64@1.2.5': + resolution: {integrity: sha512-RdvH9YongQlDE9+T2Xh5D2+dyiLHx2Gz38Af1uObyBRNWjF1qbuR51hOas0f2NFUdyA03j1+HWZCbE7yZrmI3w==} cpu: [x64] os: [darwin] @@ -4423,8 +4322,8 @@ packages: cpu: [arm64] os: [linux] - '@rspack/binding-linux-arm64-gnu@1.2.2': - resolution: {integrity: sha512-VykY/kiYOzO8E1nYzfJ9+gQEHxb5B6lt5wa8M6xFi5B6jEGU+OsaGskmAZB9/GFImeFDHxDPvhUalI4R9p8O2Q==} + '@rspack/binding-linux-arm64-gnu@1.2.5': + resolution: {integrity: sha512-jznk/CI/wN93fr8I1j3la/CAiGf8aG7ZHIpRBtT4CkNze0c5BcF3AaJVSBHVNQqgSv0qddxMt3SADpzV8rWZ6g==} cpu: [arm64] os: [linux] @@ -4433,8 +4332,8 @@ packages: cpu: [arm64] os: [linux] - '@rspack/binding-linux-arm64-musl@1.2.2': - resolution: {integrity: sha512-Z5vAC4wGfXi8XXZ6hs8Q06TYjr3zHf819HB4DI5i4C1eQTeKdZSyoFD0NHFG23bP4NWJffp8KhmoObcy9jBT5Q==} + '@rspack/binding-linux-arm64-musl@1.2.5': + resolution: {integrity: sha512-oYzcaJ0xjb1fWbbtPmjjPXeehExEgwJ8fEGYQ5TikB+p9oCLkAghnNjsz9evUhgjByxi+NTZ1YmUNwxRuQDY1Q==} cpu: [arm64] os: [linux] @@ -4443,8 +4342,8 @@ packages: cpu: [x64] os: [linux] - '@rspack/binding-linux-x64-gnu@1.2.2': - resolution: {integrity: sha512-o3pDaL+cH5EeRbDE9gZcdZpBgp5iXvYZBBhe8vZQllYgI4zN5MJEuleV7WplG3UwTXlgZg3Kht4RORSOPn96vg==} + '@rspack/binding-linux-x64-gnu@1.2.5': + resolution: {integrity: sha512-dzEKs8oi86Vi+TFRCPpgmfF5ANL0VmlZN45e1An7HipeI2C5B1xrz/H8V43vPy8XEvQuMmkXO6Sp82A0zlHvIA==} cpu: [x64] os: [linux] @@ -4453,8 +4352,8 @@ packages: cpu: [x64] os: [linux] - '@rspack/binding-linux-x64-musl@1.2.2': - resolution: {integrity: sha512-RE3e0xe4DdchHssttKzryDwjLkbrNk/4H59TkkWeGYJcLw41tmcOZVFQUOwKLUvXWVyif/vjvV/w1SMlqB4wQg==} + '@rspack/binding-linux-x64-musl@1.2.5': + resolution: {integrity: sha512-4ENeVPVSD97rRRGr6kJSm4sIPf1tKJ8vlr9hJi4sSvF7eMLWipSwIVmqRXJ2riVMRjYD2einmJ9KzI8rqQ2OwA==} cpu: [x64] os: [linux] @@ -4463,8 +4362,8 @@ packages: cpu: [arm64] os: [win32] - '@rspack/binding-win32-arm64-msvc@1.2.2': - resolution: {integrity: sha512-R+PKBYn6uzTaDdVqTHvjqiJPBr5ZHg1wg5UmFDLNH9OklzVFyQh1JInSdJRb7lzfzTRz6bEkkwUFBPQK/CGScw==} + '@rspack/binding-win32-arm64-msvc@1.2.5': + resolution: {integrity: sha512-WUoJvX/z43MWeW1JKAQIxdvqH02oLzbaGMCzIikvniZnakQovYLPH6tCYh7qD3p7uQsm+IafFddhFxTtogC3pg==} cpu: [arm64] os: [win32] @@ -4473,8 +4372,8 @@ packages: cpu: [ia32] os: [win32] - '@rspack/binding-win32-ia32-msvc@1.2.2': - resolution: {integrity: sha512-dBqz3sRAGZ2f31FgzKLDvIRfq2haRP3X3XVCT0PsiMcvt7QJng+26aYYMy2THatd/nM8IwExYeitHWeiMBoruw==} + '@rspack/binding-win32-ia32-msvc@1.2.5': + resolution: {integrity: sha512-YzPvmt/gpiacE6aAacz4dxgEbNWwoKYPaT4WYy/oITobnAui++iCFXC4IICSmlpoA1y7O8K3Qb9jbaB/lLhbwA==} cpu: [ia32] os: [win32] @@ -4483,16 +4382,16 @@ packages: cpu: [x64] os: [win32] - '@rspack/binding-win32-x64-msvc@1.2.2': - resolution: {integrity: sha512-eeAvaN831KG553cMSHkVldyk6YQn4ujgRHov6r1wtREq7CD3/ka9LMkJUepCN85K7XtwYT0N4KpFIQyf5GTGoA==} + '@rspack/binding-win32-x64-msvc@1.2.5': + resolution: {integrity: sha512-QDDshfteMZiglllm7WUh/ITemFNuexwn1Yul7cHBFGQu6HqtqKNAR0kGR8J3e15MPMlinSaygVpfRE4A0KPmjQ==} cpu: [x64] os: [win32] '@rspack/binding@1.1.1': resolution: {integrity: sha512-BRFliHbErqWrUo9X9bdik9WTRi6EgrJSQbbUiVeIYgW4gzYdfHUohgTkWo2Byu36LZolKrEjq/Uq2A8q/tc0YA==} - '@rspack/binding@1.2.2': - resolution: {integrity: sha512-GCZwpGFYlLTdJ2soPLwjw9z4LSZ+GdpbHNfBt3Cm/f/bAF8n6mZc7dHUqN893RFh7MPU17HNEL3fMw7XR+6pHg==} + '@rspack/binding@1.2.5': + resolution: {integrity: sha512-q9vQmGDFZyFVMULwOFL7488WNSgn4ue94R/njDLMMIPF4K0oEJP2QT02elfG4KVGv2CbP63D7vEFN4ZNreo/Rw==} '@rspack/cli@1.1.1': resolution: {integrity: sha512-Tm3A6Dc+gBQA67F1ShMU7c+1i3xtPBumnkwJ/TES15YaJ3iQlTehL8qzOSie5gfnWBE3Rzqyo/5t1/vg5DF8eA==} @@ -4509,8 +4408,8 @@ packages: '@swc/helpers': optional: true - '@rspack/core@1.2.2': - resolution: {integrity: sha512-EeHAmY65Uj62hSbUKesbrcWGE7jfUI887RD03G++Gj8jS4WPHEu1TFODXNOXg6pa7zyIvs2BK0Bm16Kwz8AEaQ==} + '@rspack/core@1.2.5': + resolution: {integrity: sha512-x/riOl05gOVGgGQFimBqS5i8XbUpBxPIKUC+tDX4hmNNkzxRaGpspZfNtcL+1HBMyYuoM6fOWGyCp2R290Uy6g==} engines: {node: '>=16.0.0'} peerDependencies: '@rspack/tracing': ^1.x @@ -4855,6 +4754,9 @@ packages: '@types/node@18.19.74': resolution: {integrity: sha512-HMwEkkifei3L605gFdV+/UwtpxP6JSzM+xFk2Ia6DNFSwSVBRh9qp5Tgf4lNFOMfPVuU0WnkcWpXZpgn5ufO4A==} + '@types/node@18.19.76': + resolution: {integrity: sha512-yvR7Q9LdPz2vGpmpJX5LolrgRdWvB67MJKDPSgIIzpFbaf9a1j/f5DnLp5VDyHGMR0QZHlTr1afsD87QCXFHKw==} + '@types/node@9.6.61': resolution: {integrity: sha512-/aKAdg5c8n468cYLy2eQrcR5k6chlbNwZNGUj3TboyPa2hcO2QAJcfymlqPzMiRj8B6nYKXjzQz36minFE0RwQ==} @@ -5498,6 +5400,9 @@ packages: axios@1.7.7: resolution: {integrity: sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==} + axios@1.7.9: + resolution: {integrity: sha512-LhLcE7Hbiryz8oMDdDptSrWowmB4Bl6RCt6sIJKpRB4XtVf0iEgewX3au/pJqm+Py1kCASkb/FFKjxQaLtxJvw==} + b4a@1.6.6: resolution: {integrity: sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==} @@ -5603,6 +5508,9 @@ packages: bcryptjs@2.4.3: resolution: {integrity: sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==} + better-sqlite3@11.8.1: + resolution: {integrity: sha512-9BxNaBkblMjhJW8sMRZxnxVTRgbRmssZW0Oxc1MPBTfiR+WW21e2Mk4qu8CzrcZb1LwPCnFsfDEzq+SNcBU8eg==} + better-sqlite3@8.7.0: resolution: {integrity: sha512-99jZU4le+f3G6aIl6PmmV0cxUIWqKieHxsiF7G34CVFiE+/UabpYqkU0NJIkY/96mQKikHeBjtR27vFfs5JpEw==} @@ -5781,8 +5689,8 @@ packages: caniuse-lite@1.0.30001680: resolution: {integrity: sha512-rPQy70G6AGUMnbwS1z6Xg+RkHYPAi18ihs47GH0jcxIG7wArmPgY3XbS2sRdBbxJljp3thdT8BIqv9ccCypiPA==} - caniuse-lite@1.0.30001695: - resolution: {integrity: sha512-vHyLade6wTgI2u1ec3WQBxv+2BrTERV28UXQu9LO6lZ9pYeMk34vjXFLOxo1A4UBA8XTL4njRQZdno/yYaSmWw==} + caniuse-lite@1.0.30001700: + resolution: {integrity: sha512-2S6XIXwaE7K7erT8dY+kLQcpa5ms63XlRkMkReXjle+kf6c5g38vyMl+Z5y8dSxOFDhcFe+nxnn261PLxBSQsQ==} canvas-fit@1.5.0: resolution: {integrity: sha512-onIcjRpz69/Hx5bB5HGbYKUF2uC6QT6Gp+pfpGm3A7mPfcluSLV5v4Zu+oflDUwLdUw0rLIBhUbi0v8hM4FJQQ==} @@ -5855,6 +5763,10 @@ packages: cheerio-select@2.1.0: resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} + cheerio@1.0.0: + resolution: {integrity: sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==} + engines: {node: '>=18.17'} + cheerio@1.0.0-rc.10: resolution: {integrity: sha512-g0J0q/O6mW8z5zxQ3A8E8J1hUgp4SMOvEoW/x84OwyHKe/Zccz83PVT4y5Crcr530FV6NgmKI1qvGTKVl9XXVw==} engines: {node: '>= 6'} @@ -6941,12 +6853,12 @@ packages: engines: {node: '>=0.10.0'} hasBin: true + electron-to-chromium@1.5.102: + resolution: {integrity: sha512-eHhqaja8tE/FNpIiBrvBjFV/SSKpyWHLvxuR9dPTdo+3V9ppdLmFB7ZZQ98qNovcngPLYIz0oOBF9P0FfZef5Q==} + electron-to-chromium@1.5.32: resolution: {integrity: sha512-M+7ph0VGBQqqpTT2YrabjNKSQ2fEl9PVx6AK3N558gDH9NO8O6XN9SXXFWRo9u9PbEg/bWq+tjXQr+eXmxubCw==} - electron-to-chromium@1.5.88: - resolution: {integrity: sha512-K3C2qf1o+bGzbilTDCTBhTQcMS9KW60yTAaTeeXsfvQuTDDwlokLam/AdqlqcSy9u4UainDgsHV23ksXAOgamw==} - element-size@1.1.1: resolution: {integrity: sha512-eaN+GMOq/Q+BIWy0ybsgpcYImjGIdNLyjLFJU4XsLHXYQao5jCNb36GyN6C2qwmDDYSfIBmKpPpr4VnBdLCsPQ==} @@ -6993,14 +6905,17 @@ packages: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} + encoding-sniffer@0.2.0: + resolution: {integrity: sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==} + encoding@0.1.13: resolution: {integrity: sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==} end-of-stream@1.4.4: resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} - enhanced-resolve@5.18.0: - resolution: {integrity: sha512-0/r0MySGYG8YqlayBZ6MuCfECmHFdJ5qyPh8s8wa5Hnm6SaFLSK1VYCbj+NKp090Nm1caZhD+QTnmxO7esYGyQ==} + enhanced-resolve@5.18.1: + resolution: {integrity: sha512-ZSW3ma5GkcQBIpwZTSRAI8N71Uuwgs93IezB7mf7R60tC8ZbJideoDNKjHn2O9KIlx6rkGTTEk1xUCK2E1Y2Yg==} engines: {node: '>=10.13.0'} entities@2.2.0: @@ -7069,6 +6984,10 @@ packages: resolution: {integrity: sha512-3T8uNMC3OQTHkFUsFq8r/BwAXLHvU/9O9mE0fBc/MY5iq/8H7ncvO947LmYA6ldWw9Uh8Yhf25zu6n7nML5QWQ==} engines: {node: '>= 0.4'} + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + es-shim-unscopables@1.0.2: resolution: {integrity: sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw==} @@ -7348,8 +7267,8 @@ packages: resolution: {integrity: sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==} hasBin: true - fast-xml-parser@4.5.1: - resolution: {integrity: sha512-y655CeyUQ+jj7KBbYMc4FG01V8ZQqjN+gDYGJ50RtfsUB8iG9AmwmwoAgeKLJdmueKKMrH1RJ7yXHTSoczdv5w==} + fast-xml-parser@4.5.3: + resolution: {integrity: sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig==} hasBin: true fastest-levenshtein@1.0.16: @@ -7518,8 +7437,8 @@ packages: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} engines: {node: '>= 6'} - form-data@4.0.1: - resolution: {integrity: sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==} + form-data@4.0.2: + resolution: {integrity: sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==} engines: {node: '>= 6'} formdata-node@4.4.1: @@ -7529,6 +7448,9 @@ packages: formidable@3.5.1: resolution: {integrity: sha512-WJWKelbRHN41m5dumb0/k8TeAx7Id/y3a+Z7QfhxP/htI9Js5zYaEDtG8uMgG0vM0lOlqnmjE99/kfpOYi/0Og==} + formidable@3.5.2: + resolution: {integrity: sha512-Jqc1btCy3QzRbJaICGwKcBfGWuLADRerLzDqi2NwSt/UkXLsHJw2TVResiaoBufHVHy9aSgClOHCeJsSsFLTbg==} + forwarded-for@1.1.0: resolution: {integrity: sha512-1Yam9ht7GyMXMBvuwJfUYqpdtLVodtT5ee5JMBzGiSwVVeh37ZN8LuOWkNHd6ho2zUxpSZCHuQrt1Vjl2AxDNA==} @@ -7958,6 +7880,10 @@ packages: resolution: {integrity: sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==} engines: {node: '>=8'} + hexoid@2.0.0: + resolution: {integrity: sha512-qlspKUK7IlSQv2o+5I7yhUd7TxlOG2Vr5LTa3ve2XSNVKAL/n/u/7KLvKmFNimomDIKvZFXWHv0T12mv7rT8Aw==} + engines: {node: '>=8'} + highlight-words-core@1.2.2: resolution: {integrity: sha512-BXUKIkUuh6cmmxzi5OIbUJxrG8OAk2MqoL1DtO3Wo9D2faJg2ph5ntyuQeLqaHJmzER6H5tllCDA9ZnNe9BVGg==} @@ -8040,6 +7966,9 @@ packages: htmlparser2@8.0.2: resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} + htmlparser2@9.1.0: + resolution: {integrity: sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==} + htmlparser@1.7.7: resolution: {integrity: sha512-zpK66ifkT0fauyFh2Mulrq4AqGTucxGtOhZ8OjkbSfcCpkqQEI8qRkY0tSQSJNAQ4HUZkgWaU4fK4EH6SVH9PQ==} engines: {node: '>=0.1.33'} @@ -8105,8 +8034,8 @@ packages: resolution: {integrity: sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==} engines: {node: '>=10.18'} - ibm-cloud-sdk-core@5.1.1: - resolution: {integrity: sha512-19nSrd8UcCP4q3974wtY+gxwOcD9cQfeVUkpGRWoHs4D7bN+SB5g0m5aPAPa6QjwqDY68EYkQUboEt7dTp+4jQ==} + ibm-cloud-sdk-core@5.1.3: + resolution: {integrity: sha512-FCJSK4Gf5zdmR3yEM2DDlaYDrkfhSwP3hscKzPrQEfc4/qMnFn6bZuOOw5ulr3bB/iAbfeoGF0CkIe+dWdpC7Q==} engines: {node: '>=18'} iconv-lite@0.4.24: @@ -9289,8 +9218,8 @@ packages: resolution: {integrity: sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==} engines: {node: '>=8'} - mapbox-gl@3.9.4: - resolution: {integrity: sha512-IxfpdyNzjCMzkqj/q5OUamlF1QcS+IFEARteygEgao2B8l8+UF2ahpNRgHT2EpMSE8ma1bq4LKvr+EuJ6gqniw==} + mapbox-gl@3.10.0: + resolution: {integrity: sha512-YnQxjlthuv/tidcxGYU2C8nRDVXMlAHa3qFhuOJeX4AfRP72OMRBf9ApL+M+k5VWcAXi2fcNOUVgphknjLumjA==} maplibre-gl@4.7.1: resolution: {integrity: sha512-lgL7XpIwsgICiL82ITplfS7IGwrB1OJIw/pCvprDp2dhmSSEBgmPzYRvwYYYvJGJD7fxUv1Tvpih4nZ6VrLuaA==} @@ -9556,6 +9485,9 @@ packages: napi-build-utils@1.0.2: resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} + napi-build-utils@2.0.0: + resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} + native-promise-only@0.8.1: resolution: {integrity: sha512-zkVhZUA3y8mbz652WrL5x0fB0ehrBkulWT3TomAQ9iDtyXZvzKeEA6GPxAItBYeNYl5yngKRX612qHOhvMkDeg==} @@ -9643,6 +9575,10 @@ packages: resolution: {integrity: sha512-bLn/fU/ALVBE9wj+p4Y21ZJWYFjUXLXPi/IewyLZkx3ApxKDNBWCKdReeKOtD8dWpOdDCeMyLh6ZewzcLsG2Nw==} engines: {node: '>=10'} + node-abi@3.74.0: + resolution: {integrity: sha512-c5XK0MjkGBrQPGYG24GBADZud0NCbznxNx0ZkS+ebUTrmV1qTDxPxSL8zEAPURXSbLRWVexxmP4986BziahL5w==} + engines: {node: '>=10'} + node-addon-api@6.1.0: resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} @@ -10006,6 +9942,9 @@ packages: parse5-htmlparser2-tree-adapter@7.1.0: resolution: {integrity: sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==} + parse5-parser-stream@7.1.2: + resolution: {integrity: sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==} + parse5@6.0.1: resolution: {integrity: sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==} @@ -10188,12 +10127,20 @@ packages: peerDependencies: pg: '>=8.0' + pg-pool@3.7.1: + resolution: {integrity: sha512-xIOsFoh7Vdhojas6q3596mXFsR8nwBQBXX5JiV7p9buEVAGqYL4yFzclON5P9vFrpu1u7Zwl2oriyDa89n0wbw==} + peerDependencies: + pg: '>=8.0' + pg-protocol@1.5.0: resolution: {integrity: sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ==} pg-protocol@1.7.0: resolution: {integrity: sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ==} + pg-protocol@1.7.1: + resolution: {integrity: sha512-gjTHWGYWsEgy9MsY0Gp6ZJxV24IjDqdpTW7Eh0x+WfJLFsm/TJx1MzL6T0D88mBvkpxotCQ6TwW6N+Kko7lhgQ==} + pg-types@2.2.0: resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} engines: {node: '>=4'} @@ -10211,8 +10158,8 @@ packages: pg-native: optional: true - pg@8.13.1: - resolution: {integrity: sha512-OUir1A0rPNZlX//c7ksiu7crsGZTKSOXJPgtNiHGIlC9H0lO+NC6ZDYksSgBYY/thSWhnSRBv8w1lieNNGATNQ==} + pg@8.13.3: + resolution: {integrity: sha512-P6tPt9jXbL9HVu/SSRERNYaYG++MjnscnegFh9pPHihfoBSujsrka0hyuymMzeJKFWrcG8wvCKy8rCe8e5nDUQ==} engines: {node: '>= 8.0.0'} peerDependencies: pg-native: '>=3.0.1' @@ -10285,13 +10232,13 @@ packages: resolution: {integrity: sha512-8xCNE/aT/EXKenuMDZ+xTVwkT8gsoHN2z/Q29l80u0ppGEXVvsKRzNMbtKhg8LS8k1tJLAHHylf6p4VFmP6XUQ==} engines: {node: '>= 0.4.0'} - playwright-core@1.50.0: - resolution: {integrity: sha512-CXkSSlr4JaZs2tZHI40DsZUN/NIwgaUPsyLuOAaIZp2CyF2sN5MM5NJsyB188lFSSozFxQ5fPT4qM+f0tH/6wQ==} + playwright-core@1.50.1: + resolution: {integrity: sha512-ra9fsNWayuYumt+NiM069M6OkcRb1FZSK8bgi66AtpFoWkg2+y0bJSNmkFrWhMbEBbVKC/EruAHH3g0zmtwGmQ==} engines: {node: '>=18'} hasBin: true - playwright@1.50.0: - resolution: {integrity: sha512-+GinGfGTrd2IfX1TA4N2gNmeIksSb+IAe589ZH+FlmpV3MYTx6+buChGIuDLQwrGNCw2lWibqV50fU510N7S+w==} + playwright@1.50.1: + resolution: {integrity: sha512-G8rwsOQJ63XG6BbKj2w5rHeavFjy5zynBA9zsJMMtBoe/Uf757oG12NXz6e6OirF7RCrTVAKFXbLmn1RbL7Qaw==} engines: {node: '>=18'} hasBin: true @@ -10410,6 +10357,11 @@ packages: engines: {node: '>=10'} hasBin: true + prebuild-install@7.1.3: + resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} + engines: {node: '>=10'} + hasBin: true + precond@0.2.3: resolution: {integrity: sha512-QCYG84SgGyGzqJ/vlMsxeXd/pgL/I94ixdNFyh1PusWmTCyVfPJjZ1K1jvHtsbfnXQs2TSkEP2fR7QiMZAnKFQ==} engines: {node: '>= 0.6'} @@ -10439,8 +10391,8 @@ packages: engines: {node: '>=14'} hasBin: true - prettier@3.4.2: - resolution: {integrity: sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ==} + prettier@3.5.1: + resolution: {integrity: sha512-hPpFQvHwL3Qv5AdRvBFMhnKo4tYxp0ReXiPn2bxkiohEX6mBeBwEpBSQTkD458RaaDKQMYSp4hX4UtfUTA5wDw==} engines: {node: '>=14'} hasBin: true @@ -11052,8 +11004,12 @@ packages: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} - readable-web-to-node-stream@3.0.2: - resolution: {integrity: sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==} + readable-stream@4.7.0: + resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + readable-web-to-node-stream@3.0.4: + resolution: {integrity: sha512-9nX56alTf5bwXQ3ZDipHJhusu9NTQJ/CVPtb/XHAJCXihZeitfJvIRS4GqQ/mfIoOE3IelHMrpayVrosdHBuLw==} engines: {node: '>=8'} readdirp@3.6.0: @@ -11064,8 +11020,8 @@ packages: resolution: {integrity: sha512-GkMg9uOTpIWWKbSsgwb5fA4EavTR+SG/PMPoAY8hkhHfEEY0/vqljY+XHqtDf2cr2IJtoNRDbrrEpZUiZCkYRw==} engines: {node: '>= 14.16.0'} - readdirp@4.1.1: - resolution: {integrity: sha512-h80JrZu/MHUZCyHu5ciuoI0+WxsCxzxJTILn6Fs8rxSnFPh+UVHYfeIxK1nVGugMqkfC4vJcBOYbkfkwYK0+gw==} + readdirp@4.1.2: + resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} rechoir@0.8.0: @@ -11079,6 +11035,9 @@ packages: redux@4.2.1: resolution: {integrity: sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==} + redux@5.0.1: + resolution: {integrity: sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==} + reflect-metadata@0.1.13: resolution: {integrity: sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==} @@ -11327,8 +11286,8 @@ packages: engines: {node: '>=14.0.0'} hasBin: true - sass@1.83.4: - resolution: {integrity: sha512-B1bozCeNQiOgDcLd33e2Cs2U60wZwjUUXzh900ZyQF5qUasvMdDZYbQ566LJu7cqR+sAHlAfO6RMkaID5s6qpA==} + sass@1.85.0: + resolution: {integrity: sha512-3ToiC1xZ1Y8aU7+CkgCI/tqyuPXEmYGJXO7H4uqp0xkLXUqp88rQQ4j1HmP37xSJLbCJPaIiv+cT1y+grssrww==} engines: {node: '>=14.0.0'} hasBin: true @@ -11389,6 +11348,11 @@ packages: engines: {node: '>=10'} hasBin: true + semver@7.7.1: + resolution: {integrity: sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==} + engines: {node: '>=10'} + hasBin: true + send@0.19.0: resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==} engines: {node: '>= 0.8.0'} @@ -11452,10 +11416,6 @@ packages: resolution: {integrity: sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==} engines: {node: '>=14.15.0'} - sharp@0.33.5: - resolution: {integrity: sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -11783,6 +11743,9 @@ packages: strnum@1.0.5: resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==} + strnum@1.1.1: + resolution: {integrity: sha512-O7aCHfYCamLCctjAiaucmE+fHf2DYHkus2OKCn4Wv03sykfFtgeECn505X6K4mPl8CRNd/qurC9guq+ynoN4pw==} + strongly-connected-components@1.0.1: resolution: {integrity: sha512-i0TFx4wPcO0FwX+4RkLJi1MxmcTv90jNZgxMu9XRnMXMeFUY1VJlIoXpZunPUvUUqbCT1pg5PEkFqqpcaElNaA==} @@ -11888,6 +11851,9 @@ packages: tar-fs@2.1.1: resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==} + tar-fs@2.1.2: + resolution: {integrity: sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==} + tar-fs@3.0.6: resolution: {integrity: sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==} @@ -11939,8 +11905,8 @@ packages: engines: {node: '>=10'} hasBin: true - terser@5.37.0: - resolution: {integrity: sha512-B8wRRkmre4ERucLM/uXx4MOV5cbnOlVAqUst+1+iLKPI0dOgFO28f84ptoQt9HEI537PMzfYa/d+GEPKTRXmYA==} + terser@5.39.0: + resolution: {integrity: sha512-LBAhFyLho16harJoWMg/nZsQYgTrg5jXOn2nCYjRUcZZEdE3qa2zb8QEDRUGVZBW4rlazf2fxkg8tztybTaqWw==} engines: {node: '>=10'} hasBin: true @@ -12278,6 +12244,10 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici@6.21.1: + resolution: {integrity: sha512-q/1rj5D0/zayJB2FraXdaWxbhWiNKDvu8naDT2dl1yTlvJp4BLtOcp2a5BvgGNQpYYJzau7tf1WgKv3b+7mqpQ==} + engines: {node: '>=18.17'} + unicorn-magic@0.1.0: resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==} engines: {node: '>=18'} @@ -12620,8 +12590,8 @@ packages: resolution: {integrity: sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==} engines: {node: '>=10.13.0'} - webpack@5.97.1: - resolution: {integrity: sha512-EksG6gFY3L1eFMROS/7Wzgrii5mBAFe4rIr3r2BTfo7bcc+DWwFZ4OJ/miOuHJO/A85HwyI4eQ0F6IKXesO7Fg==} + webpack@5.98.0: + resolution: {integrity: sha512-UFynvx+gM44Gv9qFgj0acCQK2VE1CtdfwFdimkapco3hlPCJ/zeq73n2yVKimVbtm+TnApIugGhLJnkU6gjYXA==} engines: {node: '>=10.13.0'} hasBin: true peerDependencies: @@ -12645,6 +12615,14 @@ packages: webworkify@1.5.0: resolution: {integrity: sha512-AMcUeyXAhbACL8S2hqqdqOLqvJ8ylmIbNwUIqQujRSouf4+eUFaXbG6F1Rbu+srlJMmxQWsiU7mOJi0nMBfM1g==} + whatwg-encoding@3.1.1: + resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} + engines: {node: '>=18'} + + whatwg-mimetype@4.0.0: + resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} + engines: {node: '>=18'} + whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} @@ -12750,6 +12728,18 @@ packages: utf-8-validate: optional: true + ws@8.18.1: + resolution: {integrity: sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + xdg-basedir@5.1.0: resolution: {integrity: sha512-GCPAHLvrIH13+c0SuacwvRYj2SxJXQ4kaVTT5xgL3kPrz56XxkF21IGhjSE1+W0aw7gpBWRGXLCPnPby6lSpmQ==} engines: {node: '>=12'} @@ -12904,6 +12894,11 @@ packages: peerDependencies: zod: ^3.24.1 + zod-to-json-schema@3.24.2: + resolution: {integrity: sha512-pNUqrcSxuuB3/+jBbU8qKUbTbDqYUaG1vf5cXFjbhGgoUuA1amO/y4Q8lzfOhHU8HNPK6VFJ18lBDKj3OHyDsg==} + peerDependencies: + zod: ^3.24.1 + zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} @@ -13009,13 +13004,13 @@ snapshots: '@anthropic-ai/sdk@0.27.3(encoding@0.1.13)': dependencies: - '@types/node': 18.19.74 + '@types/node': 18.19.76 '@types/node-fetch': 2.6.12 abort-controller: 3.0.0 agentkeepalive: 4.6.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.6.7(encoding@0.1.13) + node-fetch: 2.7.0(encoding@0.1.13) transitivePeerDependencies: - encoding @@ -13057,7 +13052,7 @@ snapshots: '@babel/compat-data@7.25.8': {} - '@babel/compat-data@7.26.5': + '@babel/compat-data@7.26.8': optional: true '@babel/core@7.25.2': @@ -13120,18 +13115,18 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/core@7.26.7': + '@babel/core@7.26.9': dependencies: '@ampproject/remapping': 2.3.0 '@babel/code-frame': 7.26.2 - '@babel/generator': 7.26.5 + '@babel/generator': 7.26.9 '@babel/helper-compilation-targets': 7.26.5 - '@babel/helper-module-transforms': 7.26.0(@babel/core@7.26.7) - '@babel/helpers': 7.26.7 - '@babel/parser': 7.26.7 - '@babel/template': 7.25.9 - '@babel/traverse': 7.26.7 - '@babel/types': 7.26.7 + '@babel/helper-module-transforms': 7.26.0(@babel/core@7.26.9) + '@babel/helpers': 7.26.9 + '@babel/parser': 7.26.9 + '@babel/template': 7.26.9 + '@babel/traverse': 7.26.9 + '@babel/types': 7.26.9 convert-source-map: 2.0.0 debug: 4.4.0(supports-color@8.1.1) gensync: 1.0.0-beta.2 @@ -13155,10 +13150,10 @@ snapshots: '@jridgewell/trace-mapping': 0.3.25 jsesc: 3.0.2 - '@babel/generator@7.26.5': + '@babel/generator@7.26.9': dependencies: - '@babel/parser': 7.26.7 - '@babel/types': 7.26.7 + '@babel/parser': 7.26.9 + '@babel/types': 7.26.9 '@jridgewell/gen-mapping': 0.3.8 '@jridgewell/trace-mapping': 0.3.25 jsesc: 3.1.0 @@ -13186,7 +13181,7 @@ snapshots: '@babel/helper-compilation-targets@7.26.5': dependencies: - '@babel/compat-data': 7.26.5 + '@babel/compat-data': 7.26.8 '@babel/helper-validator-option': 7.25.9 browserslist: 4.24.4 lru-cache: 5.1.1 @@ -13236,8 +13231,8 @@ snapshots: '@babel/helper-module-imports@7.25.9': dependencies: - '@babel/traverse': 7.26.7 - '@babel/types': 7.26.7 + '@babel/traverse': 7.26.9 + '@babel/types': 7.26.9 transitivePeerDependencies: - supports-color optional: true @@ -13272,12 +13267,12 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.26.0(@babel/core@7.26.7)': + '@babel/helper-module-transforms@7.26.0(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-module-imports': 7.25.9 '@babel/helper-validator-identifier': 7.25.9 - '@babel/traverse': 7.26.7 + '@babel/traverse': 7.26.9 transitivePeerDependencies: - supports-color optional: true @@ -13354,10 +13349,10 @@ snapshots: '@babel/template': 7.25.9 '@babel/types': 7.25.8 - '@babel/helpers@7.26.7': + '@babel/helpers@7.26.9': dependencies: - '@babel/template': 7.25.9 - '@babel/types': 7.26.7 + '@babel/template': 7.26.9 + '@babel/types': 7.26.9 optional: true '@babel/highlight@7.24.7': @@ -13393,9 +13388,9 @@ snapshots: dependencies: '@babel/types': 7.25.9 - '@babel/parser@7.26.7': + '@babel/parser@7.26.9': dependencies: - '@babel/types': 7.26.7 + '@babel/types': 7.26.9 optional: true '@babel/plugin-syntax-async-generators@7.8.4(@babel/core@7.25.8)': @@ -13403,9 +13398,9 @@ snapshots: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-async-generators@7.8.4(@babel/core@7.26.7)': + '@babel/plugin-syntax-async-generators@7.8.4(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.24.8 optional: true @@ -13414,9 +13409,9 @@ snapshots: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-bigint@7.8.3(@babel/core@7.26.7)': + '@babel/plugin-syntax-bigint@7.8.3(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.24.8 optional: true @@ -13425,9 +13420,9 @@ snapshots: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-class-properties@7.12.13(@babel/core@7.26.7)': + '@babel/plugin-syntax-class-properties@7.12.13(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.24.8 optional: true @@ -13436,9 +13431,9 @@ snapshots: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.26.7)': + '@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.24.8 optional: true @@ -13447,9 +13442,9 @@ snapshots: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.26.7)': + '@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.24.8 optional: true @@ -13468,9 +13463,9 @@ snapshots: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-logical-assignment-operators@7.10.4(@babel/core@7.26.7)': + '@babel/plugin-syntax-logical-assignment-operators@7.10.4(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.24.8 optional: true @@ -13479,9 +13474,9 @@ snapshots: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.26.7)': + '@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.24.8 optional: true @@ -13490,9 +13485,9 @@ snapshots: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.26.7)': + '@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.24.8 optional: true @@ -13501,9 +13496,9 @@ snapshots: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.26.7)': + '@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.24.8 optional: true @@ -13512,9 +13507,9 @@ snapshots: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.26.7)': + '@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.24.8 optional: true @@ -13523,9 +13518,9 @@ snapshots: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.26.7)': + '@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.24.8 optional: true @@ -13534,9 +13529,9 @@ snapshots: '@babel/core': 7.25.8 '@babel/helper-plugin-utils': 7.24.8 - '@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.26.7)': + '@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.26.9)': dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@babel/helper-plugin-utils': 7.24.8 optional: true @@ -13593,7 +13588,7 @@ snapshots: dependencies: regenerator-runtime: 0.14.1 - '@babel/runtime@7.26.7': + '@babel/runtime@7.26.9': dependencies: regenerator-runtime: 0.14.1 @@ -13609,6 +13604,13 @@ snapshots: '@babel/parser': 7.25.9 '@babel/types': 7.25.9 + '@babel/template@7.26.9': + dependencies: + '@babel/code-frame': 7.26.2 + '@babel/parser': 7.26.9 + '@babel/types': 7.26.9 + optional: true + '@babel/traverse@7.25.6': dependencies: '@babel/code-frame': 7.24.7 @@ -13645,13 +13647,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/traverse@7.26.7': + '@babel/traverse@7.26.9': dependencies: '@babel/code-frame': 7.26.2 - '@babel/generator': 7.26.5 - '@babel/parser': 7.26.7 - '@babel/template': 7.25.9 - '@babel/types': 7.26.7 + '@babel/generator': 7.26.9 + '@babel/parser': 7.26.9 + '@babel/template': 7.26.9 + '@babel/types': 7.26.9 debug: 4.4.0(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: @@ -13675,7 +13677,7 @@ snapshots: '@babel/helper-string-parser': 7.25.9 '@babel/helper-validator-identifier': 7.25.9 - '@babel/types@7.26.7': + '@babel/types@7.26.9': dependencies: '@babel/helper-string-parser': 7.25.9 '@babel/helper-validator-identifier': 7.25.9 @@ -13685,30 +13687,29 @@ snapshots: '@braintree/sanitize-url@7.1.0': {} - '@browserbasehq/sdk@2.0.0(encoding@0.1.13)': + '@browserbasehq/sdk@2.3.0(encoding@0.1.13)': dependencies: - '@types/node': 18.19.74 + '@types/node': 18.19.76 '@types/node-fetch': 2.6.12 abort-controller: 3.0.0 agentkeepalive: 4.6.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.6.7(encoding@0.1.13) + node-fetch: 2.7.0(encoding@0.1.13) transitivePeerDependencies: - encoding - '@browserbasehq/stagehand@1.10.1(@playwright/test@1.50.0)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8)': + '@browserbasehq/stagehand@1.13.0(@playwright/test@1.50.1)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8)': dependencies: '@anthropic-ai/sdk': 0.27.3(encoding@0.1.13) - '@browserbasehq/sdk': 2.0.0(encoding@0.1.13) - '@playwright/test': 1.50.0 + '@browserbasehq/sdk': 2.3.0(encoding@0.1.13) + '@playwright/test': 1.50.1 deepmerge: 4.3.1 dotenv: 16.4.7 openai: 4.78.1(encoding@0.1.13)(zod@3.23.8) - sharp: 0.33.5 - ws: 8.18.0 + ws: 8.18.1 zod: 3.23.8 - zod-to-json-schema: 3.24.1(zod@3.23.8) + zod-to-json-schema: 3.24.2(zod@3.23.8) transitivePeerDependencies: - bufferutil - encoding @@ -14010,11 +14011,6 @@ snapshots: react: 18.3.1 tslib: 2.7.0 - '@emnapi/runtime@1.3.1': - dependencies: - tslib: 2.8.1 - optional: true - '@emotion/hash@0.8.0': {} '@emotion/unitless@0.7.5': {} @@ -14200,12 +14196,12 @@ snapshots: '@humanwhocodes/object-schema@2.0.3': {} - '@ibm-cloud/watsonx-ai@1.3.2(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))': + '@ibm-cloud/watsonx-ai@1.5.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))': dependencies: '@langchain/textsplitters': 0.1.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))) - '@types/node': 18.19.74 + '@types/node': 18.19.76 extend: 3.0.2 - ibm-cloud-sdk-core: 5.1.1 + ibm-cloud-sdk-core: 5.1.3 transitivePeerDependencies: - '@langchain/core' - supports-color @@ -14228,81 +14224,6 @@ snapshots: dependencies: react: 18.3.1 - '@img/sharp-darwin-arm64@0.33.5': - optionalDependencies: - '@img/sharp-libvips-darwin-arm64': 1.0.4 - optional: true - - '@img/sharp-darwin-x64@0.33.5': - optionalDependencies: - '@img/sharp-libvips-darwin-x64': 1.0.4 - optional: true - - '@img/sharp-libvips-darwin-arm64@1.0.4': - optional: true - - '@img/sharp-libvips-darwin-x64@1.0.4': - optional: true - - '@img/sharp-libvips-linux-arm64@1.0.4': - optional: true - - '@img/sharp-libvips-linux-arm@1.0.5': - optional: true - - '@img/sharp-libvips-linux-s390x@1.0.4': - optional: true - - '@img/sharp-libvips-linux-x64@1.0.4': - optional: true - - '@img/sharp-libvips-linuxmusl-arm64@1.0.4': - optional: true - - '@img/sharp-libvips-linuxmusl-x64@1.0.4': - optional: true - - '@img/sharp-linux-arm64@0.33.5': - optionalDependencies: - '@img/sharp-libvips-linux-arm64': 1.0.4 - optional: true - - '@img/sharp-linux-arm@0.33.5': - optionalDependencies: - '@img/sharp-libvips-linux-arm': 1.0.5 - optional: true - - '@img/sharp-linux-s390x@0.33.5': - optionalDependencies: - '@img/sharp-libvips-linux-s390x': 1.0.4 - optional: true - - '@img/sharp-linux-x64@0.33.5': - optionalDependencies: - '@img/sharp-libvips-linux-x64': 1.0.4 - optional: true - - '@img/sharp-linuxmusl-arm64@0.33.5': - optionalDependencies: - '@img/sharp-libvips-linuxmusl-arm64': 1.0.4 - optional: true - - '@img/sharp-linuxmusl-x64@0.33.5': - optionalDependencies: - '@img/sharp-libvips-linuxmusl-x64': 1.0.4 - optional: true - - '@img/sharp-wasm32@0.33.5': - dependencies: - '@emnapi/runtime': 1.3.1 - optional: true - - '@img/sharp-win32-ia32@0.33.5': - optional: true - - '@img/sharp-win32-x64@0.33.5': - optional: true - '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -14689,39 +14610,39 @@ snapshots: transitivePeerDependencies: - encoding - ? '@langchain/community@0.3.24(@browserbasehq/sdk@2.0.0(encoding@0.1.13))(@browserbasehq/stagehand@1.10.1(@playwright/test@1.50.0)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.3.2(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@8.7.0)(cheerio@1.0.0-rc.10)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.1)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.1)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.1)(playwright@1.50.0)(ws@8.18.0)' + ? '@langchain/community@0.3.24(@browserbasehq/sdk@2.3.0(encoding@0.1.13))(@browserbasehq/stagehand@1.13.0(@playwright/test@1.50.1)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.5.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@11.8.1)(cheerio@1.0.0)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.3)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.3)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.3)(playwright@1.50.1)(ws@8.18.1)' : dependencies: - '@browserbasehq/stagehand': 1.10.1(@playwright/test@1.50.0)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8) - '@ibm-cloud/watsonx-ai': 1.3.2(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))) + '@browserbasehq/stagehand': 1.13.0(@playwright/test@1.50.1)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8) + '@ibm-cloud/watsonx-ai': 1.5.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))) '@langchain/core': 0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) '@langchain/openai': 0.3.17(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) binary-extensions: 2.2.0 expr-eval: 2.0.2 flat: 5.0.2 - ibm-cloud-sdk-core: 5.1.1 + ibm-cloud-sdk-core: 5.1.3 js-yaml: 4.1.0 - langchain: 0.3.11(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(cheerio@1.0.0-rc.10)(encoding@0.1.13)(handlebars@4.7.8)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) + langchain: 0.3.11(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(cheerio@1.0.0)(encoding@0.1.13)(handlebars@4.7.8)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) langsmith: 0.2.15(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) openai: 4.78.1(encoding@0.1.13)(zod@3.23.8) uuid: 10.0.0 zod: 3.23.8 zod-to-json-schema: 3.23.0(zod@3.23.8) optionalDependencies: - '@browserbasehq/sdk': 2.0.0(encoding@0.1.13) + '@browserbasehq/sdk': 2.3.0(encoding@0.1.13) '@google-ai/generativelanguage': 2.7.0(encoding@0.1.13) '@google-cloud/storage': 7.13.0(encoding@0.1.13) - better-sqlite3: 8.7.0 - cheerio: 1.0.0-rc.10 + better-sqlite3: 11.8.1 + cheerio: 1.0.0 d3-dsv: 3.0.1 - fast-xml-parser: 4.5.1 + fast-xml-parser: 4.5.3 google-auth-library: 9.14.1(encoding@0.1.13) googleapis: 137.1.0(encoding@0.1.13) ignore: 7.0.3 jsonwebtoken: 9.0.2 lodash: 4.17.21 - pg: 8.13.1 - playwright: 1.50.0 - ws: 8.18.0 + pg: 8.13.3 + playwright: 1.50.1 + ws: 8.18.1 transitivePeerDependencies: - '@langchain/anthropic' - '@langchain/aws' @@ -14883,9 +14804,9 @@ snapshots: '@mapbox/jsonlint-lines-primitives@2.0.2': {} - '@mapbox/mapbox-gl-supported@1.5.0(mapbox-gl@3.9.4)': + '@mapbox/mapbox-gl-supported@1.5.0(mapbox-gl@3.10.0)': dependencies: - mapbox-gl: 3.9.4 + mapbox-gl: 3.10.0 '@mapbox/mapbox-gl-supported@3.0.0': {} @@ -15031,7 +14952,7 @@ snapshots: '@nestjs/axios@3.0.3(@nestjs/common@10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1))(axios@1.7.4)(rxjs@7.8.1)': dependencies: '@nestjs/common': 10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1) - axios: 1.7.4(debug@4.4.0) + axios: 1.7.4 rxjs: 7.8.1 '@nestjs/common@10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1)': @@ -15175,7 +15096,7 @@ snapshots: '@nestjs/common': 10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1) '@nestjs/core': 10.4.3(@nestjs/common@10.4.3(reflect-metadata@0.1.13)(rxjs@7.8.1))(encoding@0.1.13)(reflect-metadata@0.1.13)(rxjs@7.8.1) '@nuxtjs/opencollective': 0.3.2(encoding@0.1.13) - axios: 1.7.4(debug@4.4.0) + axios: 1.7.4 chalk: 4.1.2 commander: 8.3.0 compare-versions: 4.1.4 @@ -15297,9 +15218,9 @@ snapshots: '@pkgr/core@0.1.1': {} - '@playwright/test@1.50.0': + '@playwright/test@1.50.1': dependencies: - playwright: 1.50.0 + playwright: 1.50.1 '@plotly/d3-sankey-circular@0.33.1': dependencies: @@ -15316,12 +15237,12 @@ snapshots: '@plotly/d3@3.8.2': {} - '@plotly/mapbox-gl@1.13.4(mapbox-gl@3.9.4)': + '@plotly/mapbox-gl@1.13.4(mapbox-gl@3.10.0)': dependencies: '@mapbox/geojson-rewind': 0.5.2 '@mapbox/geojson-types': 1.0.2 '@mapbox/jsonlint-lines-primitives': 2.0.2 - '@mapbox/mapbox-gl-supported': 1.5.0(mapbox-gl@3.9.4) + '@mapbox/mapbox-gl-supported': 1.5.0(mapbox-gl@3.10.0) '@mapbox/point-geometry': 0.1.0 '@mapbox/tiny-sdf': 1.2.5 '@mapbox/unitbezier': 0.0.0 @@ -15476,55 +15397,55 @@ snapshots: '@rspack/binding-darwin-arm64@1.1.1': optional: true - '@rspack/binding-darwin-arm64@1.2.2': + '@rspack/binding-darwin-arm64@1.2.5': optional: true '@rspack/binding-darwin-x64@1.1.1': optional: true - '@rspack/binding-darwin-x64@1.2.2': + '@rspack/binding-darwin-x64@1.2.5': optional: true '@rspack/binding-linux-arm64-gnu@1.1.1': optional: true - '@rspack/binding-linux-arm64-gnu@1.2.2': + '@rspack/binding-linux-arm64-gnu@1.2.5': optional: true '@rspack/binding-linux-arm64-musl@1.1.1': optional: true - '@rspack/binding-linux-arm64-musl@1.2.2': + '@rspack/binding-linux-arm64-musl@1.2.5': optional: true '@rspack/binding-linux-x64-gnu@1.1.1': optional: true - '@rspack/binding-linux-x64-gnu@1.2.2': + '@rspack/binding-linux-x64-gnu@1.2.5': optional: true '@rspack/binding-linux-x64-musl@1.1.1': optional: true - '@rspack/binding-linux-x64-musl@1.2.2': + '@rspack/binding-linux-x64-musl@1.2.5': optional: true '@rspack/binding-win32-arm64-msvc@1.1.1': optional: true - '@rspack/binding-win32-arm64-msvc@1.2.2': + '@rspack/binding-win32-arm64-msvc@1.2.5': optional: true '@rspack/binding-win32-ia32-msvc@1.1.1': optional: true - '@rspack/binding-win32-ia32-msvc@1.2.2': + '@rspack/binding-win32-ia32-msvc@1.2.5': optional: true '@rspack/binding-win32-x64-msvc@1.1.1': optional: true - '@rspack/binding-win32-x64-msvc@1.2.2': + '@rspack/binding-win32-x64-msvc@1.2.5': optional: true '@rspack/binding@1.1.1': @@ -15539,24 +15460,24 @@ snapshots: '@rspack/binding-win32-ia32-msvc': 1.1.1 '@rspack/binding-win32-x64-msvc': 1.1.1 - '@rspack/binding@1.2.2': + '@rspack/binding@1.2.5': optionalDependencies: - '@rspack/binding-darwin-arm64': 1.2.2 - '@rspack/binding-darwin-x64': 1.2.2 - '@rspack/binding-linux-arm64-gnu': 1.2.2 - '@rspack/binding-linux-arm64-musl': 1.2.2 - '@rspack/binding-linux-x64-gnu': 1.2.2 - '@rspack/binding-linux-x64-musl': 1.2.2 - '@rspack/binding-win32-arm64-msvc': 1.2.2 - '@rspack/binding-win32-ia32-msvc': 1.2.2 - '@rspack/binding-win32-x64-msvc': 1.2.2 + '@rspack/binding-darwin-arm64': 1.2.5 + '@rspack/binding-darwin-x64': 1.2.5 + '@rspack/binding-linux-arm64-gnu': 1.2.5 + '@rspack/binding-linux-arm64-musl': 1.2.5 + '@rspack/binding-linux-x64-gnu': 1.2.5 + '@rspack/binding-linux-x64-musl': 1.2.5 + '@rspack/binding-win32-arm64-msvc': 1.2.5 + '@rspack/binding-win32-ia32-msvc': 1.2.5 + '@rspack/binding-win32-x64-msvc': 1.2.5 optional: true - '@rspack/cli@1.1.1(@rspack/core@1.1.1(@swc/helpers@0.5.15))(@types/express@4.17.21)(webpack@5.97.1)': + '@rspack/cli@1.1.1(@rspack/core@1.1.1(@swc/helpers@0.5.15))(@types/express@4.17.21)(webpack@5.98.0)': dependencies: '@discoveryjs/json-ext': 0.5.7 '@rspack/core': 1.1.1(@swc/helpers@0.5.15) - '@rspack/dev-server': 1.0.9(@rspack/core@1.1.1(@swc/helpers@0.5.15))(@types/express@4.17.21)(webpack@5.97.1) + '@rspack/dev-server': 1.0.9(@rspack/core@1.1.1(@swc/helpers@0.5.15))(@types/express@4.17.21)(webpack@5.98.0) colorette: 2.0.19 exit-hook: 4.0.0 interpret: 3.1.1 @@ -15582,17 +15503,17 @@ snapshots: optionalDependencies: '@swc/helpers': 0.5.15 - '@rspack/core@1.2.2(@swc/helpers@0.5.15)': + '@rspack/core@1.2.5(@swc/helpers@0.5.15)': dependencies: '@module-federation/runtime-tools': 0.8.4 - '@rspack/binding': 1.2.2 + '@rspack/binding': 1.2.5 '@rspack/lite-tapable': 1.0.1 - caniuse-lite: 1.0.30001695 + caniuse-lite: 1.0.30001700 optionalDependencies: '@swc/helpers': 0.5.15 optional: true - '@rspack/dev-server@1.0.9(@rspack/core@1.1.1(@swc/helpers@0.5.15))(@types/express@4.17.21)(webpack@5.97.1)': + '@rspack/dev-server@1.0.9(@rspack/core@1.1.1(@swc/helpers@0.5.15))(@types/express@4.17.21)(webpack@5.98.0)': dependencies: '@rspack/core': 1.1.1(@swc/helpers@0.5.15) chokidar: 3.6.0 @@ -15601,8 +15522,8 @@ snapshots: http-proxy-middleware: 2.0.7(@types/express@4.17.21) mime-types: 2.1.35 p-retry: 4.6.2 - webpack-dev-middleware: 7.4.2(webpack@5.97.1) - webpack-dev-server: 5.0.4(webpack@5.97.1) + webpack-dev-middleware: 7.4.2(webpack@5.98.0) + webpack-dev-server: 5.0.4(webpack@5.98.0) ws: 8.18.0 transitivePeerDependencies: - '@types/express' @@ -15856,7 +15777,7 @@ snapshots: '@types/formidable@3.4.5': dependencies: - '@types/node': 18.19.50 + '@types/node': 18.19.74 '@types/geojson-vt@3.2.5': dependencies: @@ -15979,8 +15900,8 @@ snapshots: '@types/node-fetch@2.6.12': dependencies: - '@types/node': 18.19.74 - form-data: 4.0.1 + '@types/node': 18.19.76 + form-data: 4.0.2 '@types/node-forge@1.3.11': dependencies: @@ -16000,6 +15921,10 @@ snapshots: dependencies: undici-types: 5.26.5 + '@types/node@18.19.76': + dependencies: + undici-types: 5.26.5 + '@types/node@9.6.61': {} '@types/nodemailer@6.4.16': @@ -16264,9 +16189,9 @@ snapshots: '@typescript-eslint/types': 6.21.0 eslint-visitor-keys: 3.4.3 - '@uiw/react-textarea-code-editor@2.1.9(@babel/runtime@7.26.7)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + '@uiw/react-textarea-code-editor@2.1.9(@babel/runtime@7.26.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': dependencies: - '@babel/runtime': 7.26.7 + '@babel/runtime': 7.26.9 react: 18.3.1 react-dom: 18.3.1(react@18.3.1) rehype: 12.0.1 @@ -16750,7 +16675,7 @@ snapshots: awaiting@3.0.0: {} - axios@1.7.4(debug@4.4.0): + axios@1.7.4: dependencies: follow-redirects: 1.15.9(debug@4.4.0) form-data: 4.0.0 @@ -16766,6 +16691,14 @@ snapshots: transitivePeerDependencies: - debug + axios@1.7.9(debug@4.4.0): + dependencies: + follow-redirects: 1.15.9(debug@4.4.0) + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + b4a@1.6.6: {} babel-jest@29.7.0(@babel/core@7.25.8): @@ -16781,13 +16714,13 @@ snapshots: transitivePeerDependencies: - supports-color - babel-jest@29.7.0(@babel/core@7.26.7): + babel-jest@29.7.0(@babel/core@7.26.9): dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@jest/transform': 29.7.0 '@types/babel__core': 7.20.5 babel-plugin-istanbul: 6.1.1 - babel-preset-jest: 29.6.3(@babel/core@7.26.7) + babel-preset-jest: 29.6.3(@babel/core@7.26.9) chalk: 4.1.2 graceful-fs: 4.2.11 slash: 3.0.0 @@ -16795,12 +16728,12 @@ snapshots: - supports-color optional: true - babel-loader@9.2.1(@babel/core@7.25.2)(webpack@5.97.1): + babel-loader@9.2.1(@babel/core@7.25.2)(webpack@5.98.0): dependencies: '@babel/core': 7.25.2 find-cache-dir: 4.0.0 schema-utils: 4.2.0 - webpack: 5.97.1 + webpack: 5.98.0 babel-plugin-istanbul@6.1.1: dependencies: @@ -16839,21 +16772,21 @@ snapshots: '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.25.8) '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.25.8) - babel-preset-current-node-syntax@1.0.1(@babel/core@7.26.7): - dependencies: - '@babel/core': 7.26.7 - '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.26.7) - '@babel/plugin-syntax-bigint': 7.8.3(@babel/core@7.26.7) - '@babel/plugin-syntax-class-properties': 7.12.13(@babel/core@7.26.7) - '@babel/plugin-syntax-import-meta': 7.10.4(@babel/core@7.26.7) - '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.26.7) - '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.26.7) - '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.26.7) - '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.26.7) - '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.26.7) - '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.26.7) - '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.26.7) - '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.26.7) + babel-preset-current-node-syntax@1.0.1(@babel/core@7.26.9): + dependencies: + '@babel/core': 7.26.9 + '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.26.9) + '@babel/plugin-syntax-bigint': 7.8.3(@babel/core@7.26.9) + '@babel/plugin-syntax-class-properties': 7.12.13(@babel/core@7.26.9) + '@babel/plugin-syntax-import-meta': 7.10.4(@babel/core@7.26.9) + '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.26.9) + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.26.9) + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.26.9) + '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.26.9) + '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.26.9) + '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.26.9) + '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.26.9) + '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.26.9) optional: true babel-preset-jest@29.6.3(@babel/core@7.25.8): @@ -16862,11 +16795,11 @@ snapshots: babel-plugin-jest-hoist: 29.6.3 babel-preset-current-node-syntax: 1.0.1(@babel/core@7.25.8) - babel-preset-jest@29.6.3(@babel/core@7.26.7): + babel-preset-jest@29.6.3(@babel/core@7.26.9): dependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 babel-plugin-jest-hoist: 29.6.3 - babel-preset-current-node-syntax: 1.0.1(@babel/core@7.26.7) + babel-preset-current-node-syntax: 1.0.1(@babel/core@7.26.9) optional: true babel-runtime@6.26.0: @@ -16934,6 +16867,12 @@ snapshots: bcryptjs@2.4.3: {} + better-sqlite3@11.8.1: + dependencies: + bindings: 1.5.0 + prebuild-install: 7.1.3 + optional: true + better-sqlite3@8.7.0: dependencies: bindings: 1.5.0 @@ -17032,8 +16971,8 @@ snapshots: browserslist@4.24.4: dependencies: - caniuse-lite: 1.0.30001695 - electron-to-chromium: 1.5.88 + caniuse-lite: 1.0.30001700 + electron-to-chromium: 1.5.102 node-releases: 2.0.19 update-browserslist-db: 1.1.2(browserslist@4.24.4) @@ -17137,7 +17076,7 @@ snapshots: caniuse-lite@1.0.30001680: {} - caniuse-lite@1.0.30001695: {} + caniuse-lite@1.0.30001700: {} canvas-fit@1.5.0: dependencies: @@ -17224,6 +17163,21 @@ snapshots: domhandler: 5.0.3 domutils: 3.2.2 + cheerio@1.0.0: + dependencies: + cheerio-select: 2.1.0 + dom-serializer: 2.0.0 + domhandler: 5.0.3 + domutils: 3.2.2 + encoding-sniffer: 0.2.0 + htmlparser2: 9.1.0 + parse5: 7.2.1 + parse5-htmlparser2-tree-adapter: 7.1.0 + parse5-parser-stream: 7.1.2 + undici: 6.21.1 + whatwg-mimetype: 4.0.0 + optional: true + cheerio@1.0.0-rc.10: dependencies: cheerio-select: 1.6.0 @@ -17276,7 +17230,7 @@ snapshots: chokidar@4.0.3: dependencies: - readdirp: 4.1.1 + readdirp: 4.1.2 optional: true chownr@1.1.4: {} @@ -17311,10 +17265,10 @@ snapshots: clean-stack@2.2.0: {} - clean-webpack-plugin@4.0.0(webpack@5.97.1): + clean-webpack-plugin@4.0.0(webpack@5.98.0): dependencies: del: 4.1.1 - webpack: 5.97.1 + webpack: 5.98.0 clear-module@4.1.2: dependencies: @@ -17381,10 +17335,10 @@ snapshots: minimatch: 3.1.2 pkginfo: 0.4.1 - coffee-loader@3.0.0(coffeescript@2.7.0)(webpack@5.97.1): + coffee-loader@3.0.0(coffeescript@2.7.0)(webpack@5.98.0): dependencies: coffeescript: 2.7.0 - webpack: 5.97.1 + webpack: 5.98.0 coffee-react-transform@4.0.0: {} @@ -17812,7 +17766,7 @@ snapshots: css-global-keywords@1.0.1: {} - css-loader@7.1.2(@rspack/core@1.1.1(@swc/helpers@0.5.15))(webpack@5.97.1): + css-loader@7.1.2(@rspack/core@1.1.1(@swc/helpers@0.5.15))(webpack@5.98.0): dependencies: icss-utils: 5.1.0(postcss@8.4.41) postcss: 8.4.41 @@ -17824,9 +17778,9 @@ snapshots: semver: 7.6.3 optionalDependencies: '@rspack/core': 1.1.1(@swc/helpers@0.5.15) - webpack: 5.97.1 + webpack: 5.98.0 - css-loader@7.1.2(@rspack/core@1.2.2(@swc/helpers@0.5.15))(webpack@5.97.1): + css-loader@7.1.2(@rspack/core@1.2.5(@swc/helpers@0.5.15))(webpack@5.98.0): dependencies: icss-utils: 5.1.0(postcss@8.4.41) postcss: 8.4.41 @@ -17837,8 +17791,8 @@ snapshots: postcss-value-parser: 4.2.0 semver: 7.6.3 optionalDependencies: - '@rspack/core': 1.2.2(@swc/helpers@0.5.15) - webpack: 5.97.1 + '@rspack/core': 1.2.5(@swc/helpers@0.5.15) + webpack: 5.98.0 css-select@4.3.0: dependencies: @@ -18462,9 +18416,9 @@ snapshots: dependencies: jake: 10.9.2 - electron-to-chromium@1.5.32: {} + electron-to-chromium@1.5.102: {} - electron-to-chromium@1.5.88: {} + electron-to-chromium@1.5.32: {} element-size@1.1.1: {} @@ -18501,6 +18455,12 @@ snapshots: encodeurl@2.0.0: {} + encoding-sniffer@0.2.0: + dependencies: + iconv-lite: 0.6.3 + whatwg-encoding: 3.1.1 + optional: true + encoding@0.1.13: dependencies: iconv-lite: 0.6.3 @@ -18509,7 +18469,7 @@ snapshots: dependencies: once: 1.4.0 - enhanced-resolve@5.18.0: + enhanced-resolve@5.18.1: dependencies: graceful-fs: 4.2.11 tapable: 2.2.1 @@ -18629,6 +18589,13 @@ snapshots: has-tostringtag: 1.0.2 hasown: 2.0.2 + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.2.7 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + es-shim-unscopables@1.0.2: dependencies: hasown: 2.0.2 @@ -18705,10 +18672,10 @@ snapshots: string-width: 4.2.3 supports-hyperlinks: 2.2.0 - eslint-plugin-prettier@5.2.1(@types/eslint@9.6.1)(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.4.2): + eslint-plugin-prettier@5.2.1(@types/eslint@9.6.1)(eslint-config-prettier@9.1.0(eslint@8.57.1))(eslint@8.57.1)(prettier@3.5.1): dependencies: eslint: 8.57.1 - prettier: 3.4.2 + prettier: 3.5.1 prettier-linter-helpers: 1.0.0 synckit: 0.9.1 optionalDependencies: @@ -19006,9 +18973,9 @@ snapshots: dependencies: strnum: 1.0.5 - fast-xml-parser@4.5.1: + fast-xml-parser@4.5.3: dependencies: - strnum: 1.0.5 + strnum: 1.1.1 optional: true fastest-levenshtein@1.0.16: {} @@ -19067,7 +19034,7 @@ snapshots: file-type@16.5.4: dependencies: - readable-web-to-node-stream: 3.0.2 + readable-web-to-node-stream: 3.0.4 strtok3: 6.3.0 token-types: 4.2.1 @@ -19196,10 +19163,11 @@ snapshots: combined-stream: 1.0.8 mime-types: 2.1.35 - form-data@4.0.1: + form-data@4.0.2: dependencies: asynckit: 0.4.0 combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 mime-types: 2.1.35 formdata-node@4.4.1: @@ -19213,6 +19181,12 @@ snapshots: hexoid: 1.0.0 once: 1.4.0 + formidable@3.5.2: + dependencies: + dezalgo: 1.0.4 + hexoid: 2.0.0 + once: 1.4.0 + forwarded-for@1.1.0: {} forwarded@0.2.0: {} @@ -19815,6 +19789,8 @@ snapshots: hexoid@1.0.0: {} + hexoid@2.0.0: {} + highlight-words-core@1.2.2: {} history@1.17.0: @@ -19854,11 +19830,11 @@ snapshots: html-escaper@2.0.2: {} - html-loader@2.1.2(webpack@5.97.1): + html-loader@2.1.2(webpack@5.98.0): dependencies: html-minifier-terser: 5.1.1 parse5: 6.0.1 - webpack: 5.97.1 + webpack: 5.98.0 html-minifier-terser@5.1.1: dependencies: @@ -19890,7 +19866,7 @@ snapshots: html-void-elements@2.0.1: {} - html-webpack-plugin@5.6.0(@rspack/core@1.1.1(@swc/helpers@0.5.15))(webpack@5.97.1): + html-webpack-plugin@5.6.0(@rspack/core@1.1.1(@swc/helpers@0.5.15))(webpack@5.98.0): dependencies: '@types/html-minifier-terser': 6.1.0 html-minifier-terser: 6.1.0 @@ -19899,7 +19875,7 @@ snapshots: tapable: 2.2.1 optionalDependencies: '@rspack/core': 1.1.1(@swc/helpers@0.5.15) - webpack: 5.97.1 + webpack: 5.98.0 htmlparser2@6.1.0: dependencies: @@ -19929,6 +19905,14 @@ snapshots: domutils: 3.2.2 entities: 4.5.0 + htmlparser2@9.1.0: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.2.2 + entities: 4.5.0 + optional: true + htmlparser@1.7.7: {} http-deceiver@1.2.7: {} @@ -20011,12 +19995,12 @@ snapshots: hyperdyperid@1.2.0: {} - ibm-cloud-sdk-core@5.1.1: + ibm-cloud-sdk-core@5.1.3: dependencies: '@types/debug': 4.1.12 '@types/node': 10.14.22 '@types/tough-cookie': 4.0.5 - axios: 1.7.4(debug@4.4.0) + axios: 1.7.9(debug@4.4.0) camelcase: 6.3.0 debug: 4.4.0(supports-color@8.1.1) dotenv: 16.4.7 @@ -20026,7 +20010,7 @@ snapshots: isstream: 0.1.2 jsonwebtoken: 9.0.2 mime-types: 2.1.35 - retry-axios: 2.6.0(axios@1.7.4(debug@4.4.0)) + retry-axios: 2.6.0(axios@1.7.9(debug@4.4.0)) tough-cookie: 4.1.4 transitivePeerDependencies: - supports-color @@ -20078,11 +20062,11 @@ snapshots: import-meta-resolve@4.1.0: {} - imports-loader@3.1.1(webpack@5.97.1): + imports-loader@3.1.1(webpack@5.98.0): dependencies: source-map: 0.6.1 strip-comments: 2.0.1 - webpack: 5.97.1 + webpack: 5.98.0 imurmurhash@0.1.4: {} @@ -20871,7 +20855,7 @@ snapshots: jest-worker@27.5.1: dependencies: - '@types/node': 18.19.74 + '@types/node': 18.19.76 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -21122,7 +21106,7 @@ snapshots: lambda-cloud-node-api@1.0.1: {} - langchain@0.3.11(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(cheerio@1.0.0-rc.10)(encoding@0.1.13)(handlebars@4.7.8)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)): + langchain@0.3.11(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(cheerio@1.0.0)(encoding@0.1.13)(handlebars@4.7.8)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)): dependencies: '@langchain/core': 0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) '@langchain/openai': 0.3.17(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) @@ -21142,7 +21126,7 @@ snapshots: '@langchain/google-genai': 0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8) '@langchain/mistralai': 0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))) axios: 1.7.7 - cheerio: 1.0.0-rc.10 + cheerio: 1.0.0 handlebars: 4.7.8 transitivePeerDependencies: - encoding @@ -21209,10 +21193,10 @@ snapshots: '@types/node': 9.6.61 lean-client-js-core: 1.5.0 - less-loader@11.1.4(less@4.2.0)(webpack@5.97.1): + less-loader@11.1.4(less@4.2.0)(webpack@5.98.0): dependencies: less: 4.2.0 - webpack: 5.97.1 + webpack: 5.98.0 less@4.2.0: dependencies: @@ -21399,7 +21383,7 @@ snapshots: map-obj@4.3.0: {} - mapbox-gl@3.9.4: + mapbox-gl@3.10.0: dependencies: '@mapbox/jsonlint-lines-primitives': 2.0.2 '@mapbox/mapbox-gl-supported': 3.0.0 @@ -21743,6 +21727,9 @@ snapshots: napi-build-utils@1.0.2: {} + napi-build-utils@2.0.0: + optional: true + native-promise-only@0.8.1: {} nats.ws@1.30.1: @@ -21778,10 +21765,10 @@ snapshots: neo-async@2.6.2: {} - next-remove-imports@1.0.12(webpack@5.97.1): + next-remove-imports@1.0.12(webpack@5.98.0): dependencies: '@babel/core': 7.25.2 - babel-loader: 9.2.1(@babel/core@7.25.2)(webpack@5.97.1) + babel-loader: 9.2.1(@babel/core@7.25.2)(webpack@5.98.0) babel-plugin-transform-remove-imports: 1.7.0(@babel/core@7.25.2) transitivePeerDependencies: - supports-color @@ -21791,7 +21778,7 @@ snapshots: dependencies: chalk: 4.1.2 commander: 10.0.1 - formidable: 3.5.1 + formidable: 3.5.2 lodash: 4.17.21 prettier: 3.0.2 qs: 6.11.2 @@ -21801,12 +21788,12 @@ snapshots: next-tick@1.1.0: {} - next-translate@2.6.2(next@14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.83.4))(react@18.3.1): + next-translate@2.6.2(next@14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.0))(react@18.3.1): dependencies: - next: 14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.83.4) + next: 14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.0) react: 18.3.1 - next@14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.83.4): + next@14.2.22(@babel/core@7.25.2)(@opentelemetry/api@1.9.0)(@playwright/test@1.50.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.0): dependencies: '@next/env': 14.2.22 '@swc/helpers': 0.5.5 @@ -21828,8 +21815,8 @@ snapshots: '@next/swc-win32-ia32-msvc': 14.2.22 '@next/swc-win32-x64-msvc': 14.2.22 '@opentelemetry/api': 1.9.0 - '@playwright/test': 1.50.0 - sass: 1.83.4 + '@playwright/test': 1.50.1 + sass: 1.85.0 transitivePeerDependencies: - '@babel/core' - babel-plugin-macros @@ -21855,6 +21842,11 @@ snapshots: dependencies: semver: 7.6.3 + node-abi@3.74.0: + dependencies: + semver: 7.7.1 + optional: true + node-addon-api@6.1.0: {} node-addon-api@7.1.1: @@ -22309,6 +22301,11 @@ snapshots: domhandler: 5.0.3 parse5: 7.2.1 + parse5-parser-stream@7.1.2: + dependencies: + parse5: 7.2.1 + optional: true + parse5@6.0.1: {} parse5@7.2.1: @@ -22486,15 +22483,18 @@ snapshots: dependencies: pg: 8.13.0 - pg-pool@3.7.0(pg@8.13.1): + pg-pool@3.7.1(pg@8.13.3): dependencies: - pg: 8.13.1 + pg: 8.13.3 optional: true pg-protocol@1.5.0: {} pg-protocol@1.7.0: {} + pg-protocol@1.7.1: + optional: true + pg-types@2.2.0: dependencies: pg-int8: 1.0.1 @@ -22523,11 +22523,11 @@ snapshots: optionalDependencies: pg-cloudflare: 1.1.1 - pg@8.13.1: + pg@8.13.3: dependencies: pg-connection-string: 2.7.0 - pg-pool: 3.7.0(pg@8.13.1) - pg-protocol: 1.7.0 + pg-pool: 3.7.1(pg@8.13.3) + pg-protocol: 1.7.1 pg-types: 2.2.0 pgpass: 1.0.5 optionalDependencies: @@ -22588,20 +22588,20 @@ snapshots: pkginfo@0.4.1: {} - playwright-core@1.50.0: {} + playwright-core@1.50.1: {} - playwright@1.50.0: + playwright@1.50.1: dependencies: - playwright-core: 1.50.0 + playwright-core: 1.50.1 optionalDependencies: fsevents: 2.3.2 - plotly.js@2.35.2(@rspack/core@1.2.2(@swc/helpers@0.5.15))(mapbox-gl@3.9.4)(webpack@5.97.1): + plotly.js@2.35.2(@rspack/core@1.2.5(@swc/helpers@0.5.15))(mapbox-gl@3.10.0)(webpack@5.98.0): dependencies: '@plotly/d3': 3.8.2 '@plotly/d3-sankey': 0.7.2 '@plotly/d3-sankey-circular': 0.33.1 - '@plotly/mapbox-gl': 1.13.4(mapbox-gl@3.9.4) + '@plotly/mapbox-gl': 1.13.4(mapbox-gl@3.10.0) '@turf/area': 7.1.0 '@turf/bbox': 7.1.0 '@turf/centroid': 7.1.0 @@ -22612,7 +22612,7 @@ snapshots: color-parse: 2.0.0 color-rgba: 2.1.1 country-regex: 1.1.0 - css-loader: 7.1.2(@rspack/core@1.2.2(@swc/helpers@0.5.15))(webpack@5.97.1) + css-loader: 7.1.2(@rspack/core@1.2.5(@swc/helpers@0.5.15))(webpack@5.98.0) d3-force: 1.2.1 d3-format: 1.4.5 d3-geo: 1.12.1 @@ -22642,7 +22642,7 @@ snapshots: regl-scatter2d: 3.3.1 regl-splom: 1.0.14 strongly-connected-components: 1.0.1 - style-loader: 4.0.0(webpack@5.97.1) + style-loader: 4.0.0(webpack@5.98.0) superscript-text: 1.0.0 svg-path-sdf: 1.1.3 tinycolor2: 1.6.0 @@ -22764,6 +22764,22 @@ snapshots: tar-fs: 2.1.1 tunnel-agent: 0.6.0 + prebuild-install@7.1.3: + dependencies: + detect-libc: 2.0.3 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 2.0.0 + node-abi: 3.74.0 + pump: 3.0.2 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.2 + tunnel-agent: 0.6.0 + optional: true + precond@0.2.3: {} predefine@0.1.3: @@ -22782,7 +22798,7 @@ snapshots: prettier@3.3.3: {} - prettier@3.4.2: {} + prettier@3.5.1: {} pretty-error@4.0.0: dependencies: @@ -22956,11 +22972,11 @@ snapshots: raw-loader@0.5.1: {} - raw-loader@4.0.2(webpack@5.97.1): + raw-loader@4.0.2(webpack@5.98.0): dependencies: loader-utils: 2.0.4 schema-utils: 3.1.1 - webpack: 5.97.1 + webpack: 5.98.0 rc-animate@3.1.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: @@ -23453,15 +23469,15 @@ snapshots: react-lifecycles-compat@3.0.4: {} - react-plotly.js@2.6.0(plotly.js@2.35.2(@rspack/core@1.2.2(@swc/helpers@0.5.15))(mapbox-gl@3.9.4)(webpack@5.97.1))(react@18.3.1): + react-plotly.js@2.6.0(plotly.js@2.35.2(@rspack/core@1.2.5(@swc/helpers@0.5.15))(mapbox-gl@3.10.0)(webpack@5.98.0))(react@18.3.1): dependencies: - plotly.js: 2.35.2(@rspack/core@1.2.2(@swc/helpers@0.5.15))(mapbox-gl@3.9.4)(webpack@5.97.1) + plotly.js: 2.35.2(@rspack/core@1.2.5(@swc/helpers@0.5.15))(mapbox-gl@3.10.0)(webpack@5.98.0) prop-types: 15.8.1 react: 18.3.1 react-property@2.0.0: {} - react-redux@8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@4.2.1): + react-redux@8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@5.0.1): dependencies: '@babel/runtime': 7.25.6 '@types/hoist-non-react-statics': 3.3.1 @@ -23474,7 +23490,7 @@ snapshots: '@types/react': 18.3.10 '@types/react-dom': 18.3.0 react-dom: 18.3.1(react@18.3.1) - redux: 4.2.1 + redux: 5.0.1 react-refresh@0.14.2: {} @@ -23568,9 +23584,17 @@ snapshots: string_decoder: 1.3.0 util-deprecate: 1.0.2 - readable-web-to-node-stream@3.0.2: + readable-stream@4.7.0: dependencies: - readable-stream: 3.6.2 + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + readable-web-to-node-stream@3.0.4: + dependencies: + readable-stream: 4.7.0 readdirp@3.6.0: dependencies: @@ -23578,7 +23602,7 @@ snapshots: readdirp@4.0.1: {} - readdirp@4.1.1: + readdirp@4.1.2: optional: true rechoir@0.8.0: @@ -23594,6 +23618,9 @@ snapshots: dependencies: '@babel/runtime': 7.25.6 + redux@5.0.1: + optional: true + reflect-metadata@0.1.13: {} reflect.getprototypeof@1.0.6: @@ -23773,9 +23800,9 @@ snapshots: onetime: 5.1.2 signal-exit: 3.0.7 - retry-axios@2.6.0(axios@1.7.4(debug@4.4.0)): + retry-axios@2.6.0(axios@1.7.9(debug@4.4.0)): dependencies: - axios: 1.7.4(debug@4.4.0) + axios: 1.7.9(debug@4.4.0) retry-request@7.0.2(encoding@0.1.13): dependencies: @@ -23872,13 +23899,13 @@ snapshots: parse-srcset: 1.0.2 postcss: 8.4.31 - sass-loader@16.0.2(@rspack/core@1.1.1(@swc/helpers@0.5.15))(sass@1.79.3)(webpack@5.97.1): + sass-loader@16.0.2(@rspack/core@1.1.1(@swc/helpers@0.5.15))(sass@1.79.3)(webpack@5.98.0): dependencies: neo-async: 2.6.2 optionalDependencies: '@rspack/core': 1.1.1(@swc/helpers@0.5.15) sass: 1.79.3 - webpack: 5.97.1 + webpack: 5.98.0 sass@1.79.3: dependencies: @@ -23886,7 +23913,7 @@ snapshots: immutable: 4.3.7 source-map-js: 1.0.2 - sass@1.83.4: + sass@1.85.0: dependencies: chokidar: 4.0.3 immutable: 5.0.3 @@ -23955,6 +23982,9 @@ snapshots: semver@7.6.3: {} + semver@7.7.1: + optional: true + send@0.19.0: dependencies: debug: 2.6.9 @@ -24060,32 +24090,6 @@ snapshots: tar-fs: 3.0.6 tunnel-agent: 0.6.0 - sharp@0.33.5: - dependencies: - color: 4.2.3 - detect-libc: 2.0.3 - semver: 7.6.3 - optionalDependencies: - '@img/sharp-darwin-arm64': 0.33.5 - '@img/sharp-darwin-x64': 0.33.5 - '@img/sharp-libvips-darwin-arm64': 1.0.4 - '@img/sharp-libvips-darwin-x64': 1.0.4 - '@img/sharp-libvips-linux-arm': 1.0.5 - '@img/sharp-libvips-linux-arm64': 1.0.4 - '@img/sharp-libvips-linux-s390x': 1.0.4 - '@img/sharp-libvips-linux-x64': 1.0.4 - '@img/sharp-libvips-linuxmusl-arm64': 1.0.4 - '@img/sharp-libvips-linuxmusl-x64': 1.0.4 - '@img/sharp-linux-arm': 0.33.5 - '@img/sharp-linux-arm64': 0.33.5 - '@img/sharp-linux-s390x': 0.33.5 - '@img/sharp-linux-x64': 0.33.5 - '@img/sharp-linuxmusl-arm64': 0.33.5 - '@img/sharp-linuxmusl-x64': 0.33.5 - '@img/sharp-wasm32': 0.33.5 - '@img/sharp-win32-ia32': 0.33.5 - '@img/sharp-win32-x64': 0.33.5 - shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 @@ -24237,12 +24241,12 @@ snapshots: source-map-js@1.2.1: {} - source-map-loader@3.0.2(webpack@5.97.1): + source-map-loader@3.0.2(webpack@5.98.0): dependencies: abab: 2.0.6 iconv-lite: 0.6.3 source-map-js: 1.2.1 - webpack: 5.97.1 + webpack: 5.98.0 source-map-support@0.5.13: dependencies: @@ -24472,6 +24476,9 @@ snapshots: strnum@1.0.5: {} + strnum@1.1.1: + optional: true + strongly-connected-components@1.0.1: {} strtok3@6.3.0: @@ -24481,15 +24488,15 @@ snapshots: stubs@3.0.0: {} - style-loader@2.0.0(webpack@5.97.1): + style-loader@2.0.0(webpack@5.98.0): dependencies: loader-utils: 2.0.4 schema-utils: 3.3.0 - webpack: 5.97.1 + webpack: 5.98.0 - style-loader@4.0.0(webpack@5.97.1): + style-loader@4.0.0(webpack@5.98.0): dependencies: - webpack: 5.97.1 + webpack: 5.98.0 style-to-js@1.1.1: dependencies: @@ -24582,6 +24589,14 @@ snapshots: pump: 3.0.2 tar-stream: 2.2.0 + tar-fs@2.1.2: + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.2 + tar-stream: 2.2.0 + optional: true + tar-fs@3.0.6: dependencies: pump: 3.0.2 @@ -24634,25 +24649,25 @@ snapshots: mkdirp: 0.5.6 rimraf: 2.6.3 - terser-webpack-plugin@5.3.11(uglify-js@3.19.3)(webpack@5.97.1(uglify-js@3.19.3)): + terser-webpack-plugin@5.3.11(uglify-js@3.19.3)(webpack@5.98.0(uglify-js@3.19.3)): dependencies: '@jridgewell/trace-mapping': 0.3.25 jest-worker: 27.5.1 schema-utils: 4.3.0 serialize-javascript: 6.0.2 - terser: 5.37.0 - webpack: 5.97.1(uglify-js@3.19.3) + terser: 5.39.0 + webpack: 5.98.0(uglify-js@3.19.3) optionalDependencies: uglify-js: 3.19.3 - terser-webpack-plugin@5.3.11(webpack@5.97.1): + terser-webpack-plugin@5.3.11(webpack@5.98.0): dependencies: '@jridgewell/trace-mapping': 0.3.25 jest-worker: 27.5.1 schema-utils: 4.3.0 serialize-javascript: 6.0.2 - terser: 5.37.0 - webpack: 5.97.1 + terser: 5.39.0 + webpack: 5.98.0 terser@4.8.1: dependencies: @@ -24668,7 +24683,7 @@ snapshots: commander: 2.20.3 source-map-support: 0.5.21 - terser@5.37.0: + terser@5.39.0: dependencies: '@jridgewell/source-map': 0.3.6 acorn: 8.14.0 @@ -24824,7 +24839,7 @@ snapshots: '@jest/types': 29.6.3 babel-jest: 29.7.0(@babel/core@7.25.8) - ts-jest@29.2.5(@babel/core@7.26.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(jest@29.7.0(@types/node@18.19.50))(typescript@5.7.3): + ts-jest@29.2.5(@babel/core@7.26.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.9))(jest@29.7.0(@types/node@18.19.50))(typescript@5.7.3): dependencies: bs-logger: 0.2.6 ejs: 3.1.10 @@ -24838,10 +24853,10 @@ snapshots: typescript: 5.7.3 yargs-parser: 21.1.1 optionalDependencies: - '@babel/core': 7.26.7 + '@babel/core': 7.26.9 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - babel-jest: 29.7.0(@babel/core@7.26.7) + babel-jest: 29.7.0(@babel/core@7.26.9) tsd@0.22.0: dependencies: @@ -24983,6 +24998,9 @@ snapshots: undici-types@5.26.5: {} + undici@6.21.1: + optional: true + unicorn-magic@0.1.0: {} unified@10.1.2: @@ -25076,12 +25094,12 @@ snapshots: dependencies: punycode: 2.3.1 - url-loader@4.1.1(webpack@5.97.1(uglify-js@3.19.3)): + url-loader@4.1.1(webpack@5.98.0(uglify-js@3.19.3)): dependencies: loader-utils: 2.0.4 mime-types: 2.1.35 schema-utils: 3.1.1 - webpack: 5.97.1(uglify-js@3.19.3) + webpack: 5.98.0(uglify-js@3.19.3) url-parse-lax@1.0.0: dependencies: @@ -25294,7 +25312,7 @@ snapshots: - bufferutil - utf-8-validate - webpack-dev-middleware@7.4.2(webpack@5.97.1(uglify-js@3.19.3)): + webpack-dev-middleware@7.4.2(webpack@5.98.0(uglify-js@3.19.3)): dependencies: colorette: 2.0.20 memfs: 4.13.0 @@ -25303,9 +25321,9 @@ snapshots: range-parser: 1.2.1 schema-utils: 4.2.0 optionalDependencies: - webpack: 5.97.1(uglify-js@3.19.3) + webpack: 5.98.0(uglify-js@3.19.3) - webpack-dev-middleware@7.4.2(webpack@5.97.1): + webpack-dev-middleware@7.4.2(webpack@5.98.0): dependencies: colorette: 2.0.20 memfs: 4.13.0 @@ -25314,9 +25332,9 @@ snapshots: range-parser: 1.2.1 schema-utils: 4.2.0 optionalDependencies: - webpack: 5.97.1 + webpack: 5.98.0 - webpack-dev-server@5.0.4(webpack@5.97.1): + webpack-dev-server@5.0.4(webpack@5.98.0): dependencies: '@types/bonjour': 3.5.13 '@types/connect-history-api-fallback': 1.5.4 @@ -25346,10 +25364,10 @@ snapshots: serve-index: 1.9.1 sockjs: 0.3.24 spdy: 4.0.2 - webpack-dev-middleware: 7.4.2(webpack@5.97.1) + webpack-dev-middleware: 7.4.2(webpack@5.98.0) ws: 8.18.0 optionalDependencies: - webpack: 5.97.1 + webpack: 5.98.0 transitivePeerDependencies: - bufferutil - debug @@ -25368,7 +25386,7 @@ snapshots: webpack-sources@3.2.3: {} - webpack@5.97.1: + webpack@5.98.0: dependencies: '@types/eslint-scope': 3.7.7 '@types/estree': 1.0.6 @@ -25378,7 +25396,7 @@ snapshots: acorn: 8.14.0 browserslist: 4.24.4 chrome-trace-event: 1.0.4 - enhanced-resolve: 5.18.0 + enhanced-resolve: 5.18.1 es-module-lexer: 1.6.0 eslint-scope: 5.1.1 events: 3.3.0 @@ -25388,9 +25406,9 @@ snapshots: loader-runner: 4.3.0 mime-types: 2.1.35 neo-async: 2.6.2 - schema-utils: 3.3.0 + schema-utils: 4.3.0 tapable: 2.2.1 - terser-webpack-plugin: 5.3.11(webpack@5.97.1) + terser-webpack-plugin: 5.3.11(webpack@5.98.0) watchpack: 2.4.2 webpack-sources: 3.2.3 transitivePeerDependencies: @@ -25398,7 +25416,7 @@ snapshots: - esbuild - uglify-js - webpack@5.97.1(uglify-js@3.19.3): + webpack@5.98.0(uglify-js@3.19.3): dependencies: '@types/eslint-scope': 3.7.7 '@types/estree': 1.0.6 @@ -25408,7 +25426,7 @@ snapshots: acorn: 8.14.0 browserslist: 4.24.4 chrome-trace-event: 1.0.4 - enhanced-resolve: 5.18.0 + enhanced-resolve: 5.18.1 es-module-lexer: 1.6.0 eslint-scope: 5.1.1 events: 3.3.0 @@ -25418,9 +25436,9 @@ snapshots: loader-runner: 4.3.0 mime-types: 2.1.35 neo-async: 2.6.2 - schema-utils: 3.3.0 + schema-utils: 4.3.0 tapable: 2.2.1 - terser-webpack-plugin: 5.3.11(uglify-js@3.19.3)(webpack@5.97.1(uglify-js@3.19.3)) + terser-webpack-plugin: 5.3.11(uglify-js@3.19.3)(webpack@5.98.0(uglify-js@3.19.3)) watchpack: 2.4.2 webpack-sources: 3.2.3 transitivePeerDependencies: @@ -25450,6 +25468,14 @@ snapshots: webworkify@1.5.0: {} + whatwg-encoding@3.1.1: + dependencies: + iconv-lite: 0.6.3 + optional: true + + whatwg-mimetype@4.0.0: + optional: true + whatwg-url@5.0.0: dependencies: tr46: 0.0.3 @@ -25563,6 +25589,8 @@ snapshots: ws@8.18.0: {} + ws@8.18.1: {} + xdg-basedir@5.1.0: {} xml-crypto@3.2.0: @@ -25716,6 +25744,10 @@ snapshots: dependencies: zod: 3.23.8 + zod-to-json-schema@3.24.2(zod@3.23.8): + dependencies: + zod: 3.23.8 + zod@3.23.8: {} zwitch@2.0.4: {} diff --git a/src/packages/project/package.json b/src/packages/project/package.json index f6b7825753..90cd732a4d 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -36,6 +36,7 @@ "@nats-io/kv": "3.0.0-30", "@nats-io/services": "3.0.0-25", "@nteract/messaging": "^7.0.20", + "@types/formidable": "^3.4.5", "@types/lodash": "^4.14.202", "@types/primus": "^7.3.9", "@types/uuid": "^8.3.1", @@ -49,7 +50,7 @@ "expect": "^26.6.2", "express": "^4.21.2", "express-rate-limit": "^7.4.0", - "formidable": "^3.5.1", + "formidable": "^3.5.2", "get-port": "^5.1.1", "googlediff": "^0.1.0", "json-stable-stringify": "^1.0.1", From be306d4e767fada3790624606e7894fe5801b3b4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 21 Feb 2025 13:52:56 +0000 Subject: [PATCH 255/281] add env variable to not run nextjs --- src/packages/hub/hub.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/packages/hub/hub.ts b/src/packages/hub/hub.ts index 3fccbf9bfa..f7ba1f0b07 100644 --- a/src/packages/hub/hub.ts +++ b/src/packages/hub/hub.ts @@ -511,6 +511,9 @@ async function main(): Promise { program.updateDatabaseSchema = true; } + if (process.env.COCALC_DISABLE_NEXT) { + program.nextServer = false; + } //console.log("got opts", opts); From 68cbe8cd1d453c941edd58a0348234bd961bedd1 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 21 Feb 2025 13:53:10 +0000 Subject: [PATCH 256/281] nats upload: more work in progress --- src/packages/hub/servers/app/upload.ts | 66 ++++++++++++-------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/src/packages/hub/servers/app/upload.ts b/src/packages/hub/servers/app/upload.ts index 764f9c9861..ac50e21335 100644 --- a/src/packages/hub/servers/app/upload.ts +++ b/src/packages/hub/servers/app/upload.ts @@ -21,12 +21,9 @@ import { getLogger } from "@cocalc/hub/logger"; import getAccount from "@cocalc/server/auth/get-account"; import isCollaborator from "@cocalc/server/projects/is-collaborator"; import formidable from "formidable"; -import { readFile, unlink } from "fs/promises"; +import { PassThrough } from "node:stream"; -const logger = getLogger("hub:servers:app:blob-upload"); -function dbg(...args): void { - logger.debug("upload ", ...args); -} +const logger = getLogger("hub:servers:app:upload"); export default function init(router: Router) { router.post("/upload", async (req, res) => { @@ -35,7 +32,7 @@ export default function init(router: Router) { res.status(500).send("user must be signed in to upload files"); return; } - const { project_id, compute_server_id, dest, ttl, blob } = req.query; + const { project_id, compute_server_id, path, ttl, blob } = req.query; if (!blob || project_id) { if ( typeof project_id != "string" || @@ -46,46 +43,43 @@ export default function init(router: Router) { } } - dbg({ account_id, project_id, compute_server_id, dest, ttl, blob }); + logger.debug({ + account_id, + project_id, + compute_server_id, + path, + ttl, + blob, + }); try { const form = formidable({ keepExtensions: true, hashAlgorithm: "sha1", + // file = {"size":195,"newFilename":"649205cf239d49f350c645f00.py","originalFilename":"a (2).py","mimetype":"application/octet-stream","hash":"318c0246ae31424f9225b566e7e09bef6c8acc40"} + fileWriteStreamHandler: (file) => { + logger.debug("fileWriteStreamHandler", file); + const stream = new PassThrough(); + if (file == null) { + return stream; + } + + // @ts-ignore + const { originalFilename: filename, hash } = file; + (async () => { + for await (const chunk of stream) { + logger.debug("stream:", { filename, hash, chunk }); + } + })(); + + return stream; + }, }); - dbg("parsing form data..."); - // https://github.com/node-formidable/formidable?tab=readme-ov-file#parserequest-callback const [_, files] = await form.parse(req); - //dbg(`finished parsing form data. ${JSON.stringify({ fields, files })}`); - - /* Just for the sake of understanding this, this is how this looks like in the real world (formidable@3): - > files.file[0] - { - size: 80789, - filepath: '/home/hsy/p/cocalc/src/data/projects/c8787b71-a85f-437b-9d1b-29833c3a199e/asdf/asdf/8e3e4367333e45275a8d1aa03.png', - newFilename: '8e3e4367333e45275a8d1aa03.png', - mimetype: 'application/octet-stream', - mtime: '2024-04-23T09:25:53.197Z', - originalFilename: 'Screenshot from 2024-04-23 09-20-40.png' - } - */ - if (files.file?.[0] != null) { - const { filepath } = files.file[0]; - try { - dbg("got", files); - dbg("got ", await readFile(filepath)); - } finally { - try { - await unlink(filepath); - } catch (err) { - dbg("WARNING -- failed to delete uploaded file", err); - } - } - } res.send({ status: "ok", files }); } catch (err) { - dbg("upload failed ", err); + logger.debug("upload failed ", err); res.status(500).send(`upload failed -- ${err}`); } }); From 65ebf25ae1e51b9baab6861175bdbcb0b6d769c5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 21 Feb 2025 17:08:10 +0000 Subject: [PATCH 257/281] nats hub file upload: this approach sucks --- src/packages/hub/servers/app/upload.ts | 219 ++++++++++++++++++----- src/packages/nats/files/write.ts | 9 +- src/packages/project/nats/files/write.ts | 55 ++++++ 3 files changed, 236 insertions(+), 47 deletions(-) create mode 100644 src/packages/project/nats/files/write.ts diff --git a/src/packages/hub/servers/app/upload.ts b/src/packages/hub/servers/app/upload.ts index ac50e21335..f5e4be7d0c 100644 --- a/src/packages/hub/servers/app/upload.ts +++ b/src/packages/hub/servers/app/upload.ts @@ -13,15 +13,14 @@ NOTE: Code for downloading files from projects/compute servers is in the middle of packages/hub/proxy/handle-request.ts */ -// See also ./blob-upload.ts, which is similar, but targets our main -// database instead of projects, and doesn't need to worry about streaming. - import { Router } from "express"; import { getLogger } from "@cocalc/hub/logger"; import getAccount from "@cocalc/server/auth/get-account"; import isCollaborator from "@cocalc/server/projects/is-collaborator"; import formidable from "formidable"; import { PassThrough } from "node:stream"; +import { writeFile as writeFileToProject } from "@cocalc/nats/files/write"; +import { join } from "path"; const logger = getLogger("hub:servers:app:upload"); @@ -32,55 +31,183 @@ export default function init(router: Router) { res.status(500).send("user must be signed in to upload files"); return; } - const { project_id, compute_server_id, path, ttl, blob } = req.query; - if (!blob || project_id) { - if ( - typeof project_id != "string" || - !(await isCollaborator({ account_id, project_id })) - ) { - res.status(500).send("user must be collaborator on project"); - return; + const { project_id, compute_server_id, path = "", ttl, blob } = req.query; + try { + if (blob) { + await handleBlobUpload({ ttl, req, res }); + } else { + await handleUploadToProject({ + account_id, + project_id, + compute_server_id, + path, + req, + res, + }); } + } catch (err) { + logger.debug("upload failed ", err); + res.status(500).send(`upload failed -- ${err}`); } + }); +} - logger.debug({ - account_id, - project_id, - compute_server_id, - path, - ttl, - blob, - }); +async function handleBlobUpload({ ttl, req, res }) { + throw Error("blob handling not implemented"); +} - try { - const form = formidable({ - keepExtensions: true, - hashAlgorithm: "sha1", - // file = {"size":195,"newFilename":"649205cf239d49f350c645f00.py","originalFilename":"a (2).py","mimetype":"application/octet-stream","hash":"318c0246ae31424f9225b566e7e09bef6c8acc40"} - fileWriteStreamHandler: (file) => { - logger.debug("fileWriteStreamHandler", file); - const stream = new PassThrough(); - if (file == null) { - return stream; - } - - // @ts-ignore - const { originalFilename: filename, hash } = file; - (async () => { - for await (const chunk of stream) { - logger.debug("stream:", { filename, hash, chunk }); - } - })(); - - return stream; - }, +async function handleUploadToProject({ + account_id, + project_id, + compute_server_id: compute_server_id0, + path, + req, + res, +}) { + logger.debug({ + account_id, + project_id, + compute_server_id0, + path, + }); + + if ( + typeof project_id != "string" || + !(await isCollaborator({ account_id, project_id })) + ) { + throw Error("user must be collaborator on project"); + } + if (typeof compute_server_id0 != "string") { + throw Error("compute_server_id must be given"); + } + const compute_server_id = parseInt(compute_server_id0); + if (typeof path != "string") { + throw Error("path must be given"); + } + let errors: string[] = []; + + let filename = "noname.txt"; + let stream: any | null = null; + let chunkStream: any | null = null; + const form = formidable({ + keepExtensions: true, + hashAlgorithm: "sha1", + // file = {"size":195,"newFilename":"649205cf239d49f350c645f00.py","originalFilename":"a (2).py","mimetype":"application/octet-stream","hash":"318c0246ae31424f9225b566e7e09bef6c8acc40"} + fileWriteStreamHandler: (file) => { + logger.debug("fileWriteStreamHandler", file); + filename = file?.["originalFilename"] ?? "noname.txt"; + const { chunkStream: chunkStream0, totalStream } = getWriteStream({ + project_id, + compute_server_id, + path, + filename, }); + logger.debug("fileWriteStreamHandler: got back chunkstream"); + chunkStream = chunkStream0; + stream = totalStream; + return chunkStream; + }, + }); - const [_, files] = await form.parse(req); - res.send({ status: "ok", files }); - } catch (err) { - logger.debug("upload failed ", err); - res.status(500).send(`upload failed -- ${err}`); + const [fields, files] = await form.parse(req); + logger.debug("form", { fields, files }); + // fields looks like this: {"dzuuid":["ce5fa828-5155-4fa0-b30a-869bd4c956a5"],"dzchunkindex":["1"],"dztotalfilesize":["10000000"],"dzchunksize":["8000000"],"dztotalchunkcount":["2"],"dzchunkbyteoffset":["8000000"]} + + const index = parseInt(fields.dzchunkindex?.[0] ?? "0"); + const count = parseInt(fields.dztotalchunkcount?.[0] ?? "1"); + if (index == 0) { + // @ts-ignore + (async () => { + try { + logger.debug("started writing ", filename); + await writeFileToProject({ + stream, + project_id, + compute_server_id, + path: join(path, filename), + }); + logger.debug("finished writing ", filename); + } catch (err) { + errors.push(`${err}`); + } finally { + logger.debug("freeing write stream"); + freeWriteStream({ + project_id, + compute_server_id, + path, + filename, + }); + } + })(); + } + const finish = () => { + if (index == count - 1 || errors.length > 0) { + logger.debug("index = count-1, so on finish will end stream"); + if (stream) { + console.log("bytesRead", stream.bytesRead, stream.writableLength); + if (stream.writableLength > 0) { + console.log("waiting for the rest of the bytes"); + stream.once("drain", () => { + console.log("stream was drained"); + stream.end(); + }); + } else { + stream.end(); + } + } } + if (errors.length > 0) { + res.status(500).send(`upload failed -- ${errors.join(", ")}`); + } else { + res.send({ status: "ok" }); + } + }; + if (chunkStream == null) { + logger.debug("upload failed -- no chunk stream"); + res.status(500).send("upload failed -- no chunk stream"); + stream?.end(); + return; + } + if (chunkStream.closed) { + logger.debug("chunkStream already closed"); + finish(); + return; + } + logger.debug("waiting for chunkStream to end..."); + chunkStream.on("end", () => { + logger.debug("chunkStream got end"); + finish(); }); } + +function getKey(opts) { + return JSON.stringify(opts); +} + +const streams: any = {}; +export function getWriteStream(opts) { + const key = getKey(opts); + let totalStream = streams[key]; + if (totalStream == null) { + totalStream = new PassThrough(); + totalStream.bytesRead = 0; + totalStream.on("data", (chunk) => { + totalStream.bytesRead += chunk.length; + }); + + streams[key] = totalStream; + } + const chunkStream = new PassThrough(); + // make it so any write to chunkStream writes to stream: + //chunkStream.pipe(totalStream, { end: false }); + chunkStream.on("data", (data) => { + console.log("chunkstream got data", data); + totalStream.write(data); + }); + + return { chunkStream, totalStream }; +} + +function freeWriteStream(opts) { + delete streams[getKey(opts)]; +} diff --git a/src/packages/nats/files/write.ts b/src/packages/nats/files/write.ts index 0ee189b9b2..3bd7f3e76a 100644 --- a/src/packages/nats/files/write.ts +++ b/src/packages/nats/files/write.ts @@ -97,6 +97,13 @@ export async function createServer({ project_id, compute_server_id, createWriteStream, +}: { + project_id: string; + compute_server_id: number; + // createWriteStream returns a writeable stream + // for writing the specified path to disk. It + // can be an async function. + createWriteStream: (path: string) => any; }) { const subject = getWriteSubject({ project_id, compute_server_id }); let sub = subs[subject]; @@ -134,7 +141,7 @@ async function handleMessage({ const { jc } = await getEnv(); try { const { path, name, maxWait } = jc.decode(mesg.data); - const writeStream = createWriteStream(path); + const writeStream = await createWriteStream(path); writeStream.on("error", (err) => { error = `${err}`; mesg.respond(jc.encode({ error, status: "error" })); diff --git a/src/packages/project/nats/files/write.ts b/src/packages/project/nats/files/write.ts new file mode 100644 index 0000000000..b4dc2728c6 --- /dev/null +++ b/src/packages/project/nats/files/write.ts @@ -0,0 +1,55 @@ +/* + +DEVELOPMENT: + + +1. Stop the files:write service running in the project by running this in your browser: + + await cc.client.nats_client.projectApi(cc.current()).system.terminate({service:'files:write'}) + + {status: 'terminated', service: 'files:write'} + +You can also skip step 1 if you instead set COMPUTE_SERVER_ID to something nonzero... + +2. Setup the project environment variables. Then start the server in node: + + + ~/cocalc/src/packages/project/nats$ . project-env.sh + $ node + Welcome to Node.js v18.17.1. + Type ".help" for more information. + + require('@cocalc/project/nats/files/write').init() + + +*/ + +import "@cocalc/project/nats/env"; // ensure nats env available +import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; +import { createWriteStream as fs_createWriteStream } from "fs"; +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { join } from "path"; +import { + createServer, + close as closeWriteServer, +} from "@cocalc/nats/files/write"; + +async function createWriteStream(path: string) { + if (path[0] != "/" && process.env.HOME) { + path = join(process.env.HOME, path); + } + await ensureContainingDirectoryExists(path); + const stream = fs_createWriteStream(path); + // TODO: path should be a temporary path to indicate that it is a partial + // upload, then get moved to path when done or deleted on error. + return stream; +} + +// the project should call this on startup: +export async function init() { + await createServer({ project_id, compute_server_id, createWriteStream }); +} + +export async function close() { + await closeWriteServer({ project_id, compute_server_id }); +} From 3d024868ae03a74a3ec65baaff72e31b2fdf2c5f Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 21 Feb 2025 19:06:47 +0000 Subject: [PATCH 258/281] nats file upload: this works (but is still not pretty) --- src/packages/hub/servers/app/upload.ts | 83 ++++++++---------------- src/packages/nats/files/read.ts | 22 +++++-- src/packages/nats/files/write.ts | 5 ++ src/packages/project/nats/files/write.ts | 11 ++++ 4 files changed, 61 insertions(+), 60 deletions(-) diff --git a/src/packages/hub/servers/app/upload.ts b/src/packages/hub/servers/app/upload.ts index f5e4be7d0c..9e7ea31ec9 100644 --- a/src/packages/hub/servers/app/upload.ts +++ b/src/packages/hub/servers/app/upload.ts @@ -21,6 +21,7 @@ import formidable from "formidable"; import { PassThrough } from "node:stream"; import { writeFile as writeFileToProject } from "@cocalc/nats/files/write"; import { join } from "path"; +import { delay } from "awaiting"; const logger = getLogger("hub:servers:app:upload"); @@ -86,6 +87,7 @@ async function handleUploadToProject({ } let errors: string[] = []; + const state = { done: false }; let filename = "noname.txt"; let stream: any | null = null; let chunkStream: any | null = null; @@ -94,7 +96,7 @@ async function handleUploadToProject({ hashAlgorithm: "sha1", // file = {"size":195,"newFilename":"649205cf239d49f350c645f00.py","originalFilename":"a (2).py","mimetype":"application/octet-stream","hash":"318c0246ae31424f9225b566e7e09bef6c8acc40"} fileWriteStreamHandler: (file) => { - logger.debug("fileWriteStreamHandler", file); + console.log("fileWriteStreamHandler"); filename = file?.["originalFilename"] ?? "noname.txt"; const { chunkStream: chunkStream0, totalStream } = getWriteStream({ project_id, @@ -102,35 +104,44 @@ async function handleUploadToProject({ path, filename, }); - logger.debug("fileWriteStreamHandler: got back chunkstream"); + console.log("fileWriteStreamHandler: got back chunkstream"); chunkStream = chunkStream0; stream = totalStream; + (async () => { + for await (const data of chunkStream) { + stream.write(data); + } + state.done = true; + })(); return chunkStream; }, }); - const [fields, files] = await form.parse(req); - logger.debug("form", { fields, files }); + console.log('starting parsing form"'); + const [fields] = await form.parse(req); + // console.log("form", { fields, files }); // fields looks like this: {"dzuuid":["ce5fa828-5155-4fa0-b30a-869bd4c956a5"],"dzchunkindex":["1"],"dztotalfilesize":["10000000"],"dzchunksize":["8000000"],"dztotalchunkcount":["2"],"dzchunkbyteoffset":["8000000"]} const index = parseInt(fields.dzchunkindex?.[0] ?? "0"); const count = parseInt(fields.dztotalchunkcount?.[0] ?? "1"); if (index == 0) { + // start // @ts-ignore (async () => { try { - logger.debug("started writing ", filename); + console.log("NATS: started writing ", filename); await writeFileToProject({ stream, project_id, compute_server_id, - path: join(path, filename), + path: `${join(path, filename)}.partial-${Math.random()}`, }); - logger.debug("finished writing ", filename); + console.log("NATS: finished writing ", filename); } catch (err) { + console.log("NATS: error ", err); errors.push(`${err}`); } finally { - logger.debug("freeing write stream"); + console.log("NATS: freeing write stream"); freeWriteStream({ project_id, compute_server_id, @@ -140,44 +151,18 @@ async function handleUploadToProject({ } })(); } - const finish = () => { - if (index == count - 1 || errors.length > 0) { - logger.debug("index = count-1, so on finish will end stream"); - if (stream) { - console.log("bytesRead", stream.bytesRead, stream.writableLength); - if (stream.writableLength > 0) { - console.log("waiting for the rest of the bytes"); - stream.once("drain", () => { - console.log("stream was drained"); - stream.end(); - }); - } else { - stream.end(); - } - } - } - if (errors.length > 0) { - res.status(500).send(`upload failed -- ${errors.join(", ")}`); - } else { - res.send({ status: "ok" }); + if (index == count - 1) { + console.log("finish"); + while (!state.done) { + await delay(100); } - }; - if (chunkStream == null) { - logger.debug("upload failed -- no chunk stream"); - res.status(500).send("upload failed -- no chunk stream"); - stream?.end(); - return; + stream.end(); } - if (chunkStream.closed) { - logger.debug("chunkStream already closed"); - finish(); - return; + if (errors.length > 0) { + res.status(500).send(`upload failed -- ${errors.join(", ")}`); + } else { + res.send({ status: "ok" }); } - logger.debug("waiting for chunkStream to end..."); - chunkStream.on("end", () => { - logger.debug("chunkStream got end"); - finish(); - }); } function getKey(opts) { @@ -190,21 +175,9 @@ export function getWriteStream(opts) { let totalStream = streams[key]; if (totalStream == null) { totalStream = new PassThrough(); - totalStream.bytesRead = 0; - totalStream.on("data", (chunk) => { - totalStream.bytesRead += chunk.length; - }); - streams[key] = totalStream; } const chunkStream = new PassThrough(); - // make it so any write to chunkStream writes to stream: - //chunkStream.pipe(totalStream, { end: false }); - chunkStream.on("data", (data) => { - console.log("chunkstream got data", data); - totalStream.write(data); - }); - return { chunkStream, totalStream }; } diff --git a/src/packages/nats/files/read.ts b/src/packages/nats/files/read.ts index 4d226cfc96..e6229c8205 100644 --- a/src/packages/nats/files/read.ts +++ b/src/packages/nats/files/read.ts @@ -106,18 +106,30 @@ async function handleMessage(mesg, createReadStream) { } } +const MAX_NATS_CHUNK_SIZE = 16384 * 16 * 3; + +function getSeqHeader(seq) { + const h = headers(); + h.append("seq", `${seq}`); + return { headers: h }; +} + async function sendData(mesg, createReadStream) { const { jc } = await getEnv(); const { path } = jc.decode(mesg.data); let seq = 0; - for await (const chunk of createReadStream(path, { + for await (let chunk of createReadStream(path, { highWaterMark: 16384 * 16 * 3, })) { - const h = headers(); - seq += 1; - h.append("seq", `${seq}`); // console.log("sending ", { seq, bytes: chunk.length }); - mesg.respond(chunk, { headers: h }); + // We must break the chunk into smaller messages or it will + // get bounced by nats... TODO: can we get the max + // message size from nats? + while (chunk.length > 0) { + seq += 1; + mesg.respond(chunk.slice(0, MAX_NATS_CHUNK_SIZE), getSeqHeader(seq)); + chunk = chunk.slice(MAX_NATS_CHUNK_SIZE); + } } } diff --git a/src/packages/nats/files/write.ts b/src/packages/nats/files/write.ts index 3bd7f3e76a..d8f6e0f5c0 100644 --- a/src/packages/nats/files/write.ts +++ b/src/packages/nats/files/write.ts @@ -142,6 +142,7 @@ async function handleMessage({ try { const { path, name, maxWait } = jc.decode(mesg.data); const writeStream = await createWriteStream(path); + console.log("created writeStream"); writeStream.on("error", (err) => { error = `${err}`; mesg.respond(jc.encode({ error, status: "error" })); @@ -157,12 +158,16 @@ async function handleMessage({ maxWait, })) { if (error) { + console.log("error", error); + writeStream.end(); return; } writeStream.write(chunk); chunks += 1; bytes += chunk.length; + console.log("wrote ", bytes); } + console.log("ended write stream"); writeStream.end(); mesg.respond(jc.encode({ status: "success", bytes, chunks })); } catch (err) { diff --git a/src/packages/project/nats/files/write.ts b/src/packages/project/nats/files/write.ts index b4dc2728c6..a97ce062e1 100644 --- a/src/packages/project/nats/files/write.ts +++ b/src/packages/project/nats/files/write.ts @@ -35,11 +35,22 @@ import { } from "@cocalc/nats/files/write"; async function createWriteStream(path: string) { + console.log("createWriteStream", { path }); if (path[0] != "/" && process.env.HOME) { path = join(process.env.HOME, path); } await ensureContainingDirectoryExists(path); const stream = fs_createWriteStream(path); + stream.on("ready", (data) => { + console.log("ready "); + }); + stream.on("close", () => { + console.log("done", stream.bytesWritten); + }); + setTimeout(() => { + console.log("bytes", stream.bytesWritten); + }, 1000); + // TODO: path should be a temporary path to indicate that it is a partial // upload, then get moved to path when done or deleted on error. return stream; From 4b142f31a923c55eb9ca755d47da9742cfde5681 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 21 Feb 2025 19:13:20 +0000 Subject: [PATCH 259/281] nats hub upload: I think this is the hard part, done. --- src/packages/hub/servers/app/upload.ts | 28 ++++++++++++------------ src/packages/nats/files/write.ts | 8 +++---- src/packages/project/nats/files/write.ts | 11 +--------- 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/src/packages/hub/servers/app/upload.ts b/src/packages/hub/servers/app/upload.ts index 9e7ea31ec9..09cb819246 100644 --- a/src/packages/hub/servers/app/upload.ts +++ b/src/packages/hub/servers/app/upload.ts @@ -21,7 +21,6 @@ import formidable from "formidable"; import { PassThrough } from "node:stream"; import { writeFile as writeFileToProject } from "@cocalc/nats/files/write"; import { join } from "path"; -import { delay } from "awaiting"; const logger = getLogger("hub:servers:app:upload"); @@ -87,7 +86,7 @@ async function handleUploadToProject({ } let errors: string[] = []; - const state = { done: false }; + const done = { state: false, cb: () => {} }; let filename = "noname.txt"; let stream: any | null = null; let chunkStream: any | null = null; @@ -96,7 +95,6 @@ async function handleUploadToProject({ hashAlgorithm: "sha1", // file = {"size":195,"newFilename":"649205cf239d49f350c645f00.py","originalFilename":"a (2).py","mimetype":"application/octet-stream","hash":"318c0246ae31424f9225b566e7e09bef6c8acc40"} fileWriteStreamHandler: (file) => { - console.log("fileWriteStreamHandler"); filename = file?.["originalFilename"] ?? "noname.txt"; const { chunkStream: chunkStream0, totalStream } = getWriteStream({ project_id, @@ -104,20 +102,19 @@ async function handleUploadToProject({ path, filename, }); - console.log("fileWriteStreamHandler: got back chunkstream"); chunkStream = chunkStream0; stream = totalStream; (async () => { for await (const data of chunkStream) { stream.write(data); } - state.done = true; + done.state = true; + done.cb(); })(); return chunkStream; }, }); - console.log('starting parsing form"'); const [fields] = await form.parse(req); // console.log("form", { fields, files }); // fields looks like this: {"dzuuid":["ce5fa828-5155-4fa0-b30a-869bd4c956a5"],"dzchunkindex":["1"],"dztotalfilesize":["10000000"],"dzchunksize":["8000000"],"dztotalchunkcount":["2"],"dzchunkbyteoffset":["8000000"]} @@ -129,19 +126,19 @@ async function handleUploadToProject({ // @ts-ignore (async () => { try { - console.log("NATS: started writing ", filename); + // console.log("NATS: started writing ", filename); await writeFileToProject({ stream, project_id, compute_server_id, path: `${join(path, filename)}.partial-${Math.random()}`, }); - console.log("NATS: finished writing ", filename); + // console.log("NATS: finished writing ", filename); } catch (err) { - console.log("NATS: error ", err); + // console.log("NATS: error ", err); errors.push(`${err}`); } finally { - console.log("NATS: freeing write stream"); + // console.log("NATS: freeing write stream"); freeWriteStream({ project_id, compute_server_id, @@ -152,11 +149,14 @@ async function handleUploadToProject({ })(); } if (index == count - 1) { - console.log("finish"); - while (!state.done) { - await delay(100); + // console.log("finish"); + if (!done.state) { + done.cb = () => { + stream.end(); + }; + } else { + stream.end(); } - stream.end(); } if (errors.length > 0) { res.status(500).send(`upload failed -- ${errors.join(", ")}`); diff --git a/src/packages/nats/files/write.ts b/src/packages/nats/files/write.ts index d8f6e0f5c0..20a6465a36 100644 --- a/src/packages/nats/files/write.ts +++ b/src/packages/nats/files/write.ts @@ -142,7 +142,7 @@ async function handleMessage({ try { const { path, name, maxWait } = jc.decode(mesg.data); const writeStream = await createWriteStream(path); - console.log("created writeStream"); + // console.log("created writeStream"); writeStream.on("error", (err) => { error = `${err}`; mesg.respond(jc.encode({ error, status: "error" })); @@ -158,16 +158,16 @@ async function handleMessage({ maxWait, })) { if (error) { - console.log("error", error); + // console.log("error", error); writeStream.end(); return; } writeStream.write(chunk); chunks += 1; bytes += chunk.length; - console.log("wrote ", bytes); + // console.log("wrote ", bytes); } - console.log("ended write stream"); + // console.log("ended write stream"); writeStream.end(); mesg.respond(jc.encode({ status: "success", bytes, chunks })); } catch (err) { diff --git a/src/packages/project/nats/files/write.ts b/src/packages/project/nats/files/write.ts index a97ce062e1..39e2e507a9 100644 --- a/src/packages/project/nats/files/write.ts +++ b/src/packages/project/nats/files/write.ts @@ -35,21 +35,12 @@ import { } from "@cocalc/nats/files/write"; async function createWriteStream(path: string) { - console.log("createWriteStream", { path }); + // console.log("createWriteStream", { path }); if (path[0] != "/" && process.env.HOME) { path = join(process.env.HOME, path); } await ensureContainingDirectoryExists(path); const stream = fs_createWriteStream(path); - stream.on("ready", (data) => { - console.log("ready "); - }); - stream.on("close", () => { - console.log("done", stream.bytesWritten); - }); - setTimeout(() => { - console.log("bytes", stream.bytesWritten); - }, 1000); // TODO: path should be a temporary path to indicate that it is a partial // upload, then get moved to path when done or deleted on error. From 91c36da0dd6c1c89edbe5b741384bd49e3647003 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 22 Feb 2025 15:54:28 +0000 Subject: [PATCH 260/281] project: completely remove raw file server! (major security enhancement); rename files on upload; fix some instances of "raw" remaining in frontend code; less verbose jupyter pool messages --- .../frontend/misc/process-links/generic.ts | 6 +- src/packages/frontend/project/utils.ts | 18 +- src/packages/hub/servers/app/upload.ts | 12 +- src/packages/jupyter/pool/pool.ts | 18 +- src/packages/nats/files/write.ts | 4 +- src/packages/pnpm-lock.yaml | 203 ++------------- src/packages/project/nats/files/write.ts | 14 +- src/packages/project/package.json | 4 +- .../project/servers/browser/http-server.ts | 16 -- .../project/servers/browser/static.ts | 31 --- src/packages/project/upload.ts | 241 ------------------ src/scripts/g.sh | 4 + 12 files changed, 78 insertions(+), 493 deletions(-) delete mode 100644 src/packages/project/servers/browser/static.ts delete mode 100644 src/packages/project/upload.ts diff --git a/src/packages/frontend/misc/process-links/generic.ts b/src/packages/frontend/misc/process-links/generic.ts index c1fc5327d2..33ea9e33d5 100644 --- a/src/packages/frontend/misc/process-links/generic.ts +++ b/src/packages/frontend/misc/process-links/generic.ts @@ -13,9 +13,9 @@ Define a jQuery plugin that processes links. import { join } from "path"; import { is_valid_uuid_string as isUUID } from "@cocalc/util/misc"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import { isCoCalcURL } from "@cocalc/frontend/lib/cocalc-urls"; import Fragment, { FragmentId } from "@cocalc/frontend/misc/fragment-id"; +import { fileURL } from "@cocalc/frontend/lib/cocalc-urls"; type jQueryAPI = Function; @@ -199,7 +199,7 @@ function processMediaTag( // j-i should be 36, unless we ever start to have different (vanity) project_ids const path = src.slice(j + "/files/".length); projectId = src.slice(i + "/projects/".length, j); - newSrc = join(appBasePath, projectId, "raw", path); + newSrc = fileURL({ project_id: projectId, path }); y.attr(attr, newSrc); return; } @@ -209,7 +209,7 @@ function processMediaTag( } // we do not have an absolute url, hence we assume it is a // relative URL to a file in a project - newSrc = join(appBasePath, opts.projectId, "raw", opts.filePath, src); + newSrc = `${fileURL({ project_id: opts.projectId, path: opts.filePath })}/${src}`; } if (newSrc != null) { y.attr(attr, newSrc); diff --git a/src/packages/frontend/project/utils.ts b/src/packages/frontend/project/utils.ts index 9ed7ac5038..2e9837ddd5 100644 --- a/src/packages/frontend/project/utils.ts +++ b/src/packages/frontend/project/utils.ts @@ -8,7 +8,6 @@ import * as dogNames from "dog-names"; import * as os_path from "path"; import { generate as heroku } from "project-name-generator"; import * as superb from "superb"; -import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import { file_options } from "@cocalc/frontend/editor-tmp"; import { BASE_URL } from "@cocalc/frontend/misc"; import { webapp_client } from "@cocalc/frontend/webapp-client"; @@ -25,6 +24,7 @@ import { unreachable, uuid, } from "@cocalc/util/misc"; +import { fileURL } from "@cocalc/frontend/lib/cocalc-urls"; export function randomPetName() { return Math.random() > 0.5 ? catNames.random() : dogNames.allRandom(); @@ -290,13 +290,21 @@ export function url_fullpath(project_id: string, path: string): string { } // returns the URL for the file at the given path -export function url_href(project_id: string, path: string): string { - return os_path.join(appBasePath, project_id, "raw", encode_path(path)); +export function url_href( + project_id: string, + path: string, + compute_server_id?: number, +): string { + return fileURL({ project_id, path, compute_server_id }); } // returns the download URL for a file at a given path -export function download_href(project_id: string, path: string): string { - return `${url_href(project_id, path)}?download`; +export function download_href( + project_id: string, + path: string, + compute_server_id?: number, +): string { + return `${url_href(project_id, path, compute_server_id)}?download`; } export function in_snapshot_path(path: string): boolean { diff --git a/src/packages/hub/servers/app/upload.ts b/src/packages/hub/servers/app/upload.ts index 09cb819246..cc6d3ce46f 100644 --- a/src/packages/hub/servers/app/upload.ts +++ b/src/packages/hub/servers/app/upload.ts @@ -34,7 +34,9 @@ export default function init(router: Router) { const { project_id, compute_server_id, path = "", ttl, blob } = req.query; try { if (blob) { - await handleBlobUpload({ ttl, req, res }); + //await handleBlobUpload({ ttl, req, res }); + console.log(ttl); + throw Error("not implemented"); } else { await handleUploadToProject({ account_id, @@ -52,9 +54,9 @@ export default function init(router: Router) { }); } -async function handleBlobUpload({ ttl, req, res }) { - throw Error("blob handling not implemented"); -} +// async function handleBlobUpload({ ttl, req, res }) { +// throw Error("blob handling not implemented"); +// } async function handleUploadToProject({ account_id, @@ -131,7 +133,7 @@ async function handleUploadToProject({ stream, project_id, compute_server_id, - path: `${join(path, filename)}.partial-${Math.random()}`, + path: join(path, filename), }); // console.log("NATS: finished writing ", filename); } catch (err) { diff --git a/src/packages/jupyter/pool/pool.ts b/src/packages/jupyter/pool/pool.ts index f63af4cf65..9aaba71cf4 100644 --- a/src/packages/jupyter/pool/pool.ts +++ b/src/packages/jupyter/pool/pool.ts @@ -127,7 +127,7 @@ export default async function launchJupyterKernel( } const key = makeKey({ name, opts }); - log.debug("launchJupyterKernel", key); + log.debug("launchJupyterKernel", key.slice(0, 30)); try { if (POOL[key] == null) { POOL[key] = []; @@ -160,21 +160,21 @@ const replenishPool = reuseInFlight( if (isBlacklisted(name)) { log.debug( "replenishPool", - key, + key.slice(0, 30), ` -- skipping since ${name} is blacklisted`, ); return; } const size: number = size_arg ?? getSize(); const timeout_s: number = timeout_s_arg ?? getTimeoutS(); - log.debug("replenishPool", key, { size, timeout_s }); + log.debug("replenishPool", key.slice(0, 30), { size, timeout_s }); try { if (POOL[key] == null) { POOL[key] = []; } const pool = POOL[key]; while (pool.length < size) { - log.debug("replenishPool - creating a kernel", key); + log.debug("replenishPool - creating a kernel", key.slice(0, 30)); writeConfig(key); await delay(getLaunchDelayMS()); const kernel = await launchJupyterKernelNoPool(name, opts); @@ -182,7 +182,11 @@ const replenishPool = reuseInFlight( EXPIRE[key] = Math.max(EXPIRE[key] ?? 0, Date.now() + 1000 * timeout_s); } } catch (error) { - log.error("Failed to replenish Jupyter kernel pool", error); + log.error( + "Failed to replenish Jupyter kernel pool", + key.slice(0, 30), + error, + ); throw error; } }, @@ -219,11 +223,11 @@ async function fillWhenEmpty() { } async function maintainPool() { - log.debug("maintainPool", { EXPIRE }); + log.debug("maintainPool"); const now = Date.now(); for (const key in EXPIRE) { if (EXPIRE[key] < now) { - log.debug("maintainPool -- expiring key=", key); + log.debug("maintainPool -- expiring key=", key.slice(0, 30)); const pool = POOL[key] ?? []; while (pool.length > 0) { const kernel = pool.shift() as SpawnedKernel; diff --git a/src/packages/nats/files/write.ts b/src/packages/nats/files/write.ts index 20a6465a36..35ddc51c91 100644 --- a/src/packages/nats/files/write.ts +++ b/src/packages/nats/files/write.ts @@ -147,6 +147,7 @@ async function handleMessage({ error = `${err}`; mesg.respond(jc.encode({ error, status: "error" })); console.warn(`error writing ${path}: ${error}`); + writeStream.emit("remove") }); let chunks = 0; let bytes = 0; @@ -167,8 +168,9 @@ async function handleMessage({ bytes += chunk.length; // console.log("wrote ", bytes); } - // console.log("ended write stream"); + console.log("ending write stream"); writeStream.end(); + writeStream.emit("rename") mesg.respond(jc.encode({ status: "success", bytes, chunks })); } catch (err) { if (!error) { diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 62cb156a71..5e65fe9624 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -612,7 +612,7 @@ importers: version: 2.6.0(plotly.js@2.35.2(@rspack/core@1.2.5(@swc/helpers@0.5.15))(mapbox-gl@3.10.0)(webpack@5.98.0))(react@18.3.1) react-redux: specifier: ^8.0.5 - version: 8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@5.0.1) + version: 8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@4.2.1) react-timeago: specifier: ^7.2.0 version: 7.2.0(react@18.3.1) @@ -1320,9 +1320,6 @@ importers: '@nteract/messaging': specifier: ^7.0.20 version: 7.0.20 - '@types/formidable': - specifier: ^3.4.5 - version: 3.4.5 '@types/lodash': specifier: ^4.14.202 version: 4.17.9 @@ -1349,7 +1346,7 @@ importers: version: 3.0.0 debug: specifier: ^4.4.0 - version: 4.4.0(supports-color@8.1.1) + version: 4.4.0 diskusage: specifier: ^1.1.3 version: 1.2.0 @@ -1362,9 +1359,6 @@ importers: express-rate-limit: specifier: ^7.4.0 version: 7.4.0(express@4.21.2) - formidable: - specifier: ^3.5.1 - version: 3.5.1 get-port: specifier: ^5.1.1 version: 5.1.1 @@ -1404,9 +1398,9 @@ importers: prom-client: specifier: ^13.0.0 version: 13.2.0 - serve-index: - specifier: ^1.9.1 - version: 1.9.1 + rimraf: + specifier: ^5.0.5 + version: 5.0.10 temp: specifier: ^0.9.4 version: 0.9.4 @@ -1494,7 +1488,7 @@ importers: version: 0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) '@langchain/community': specifier: ^0.3.24 - version: 0.3.24(@browserbasehq/sdk@2.3.0(encoding@0.1.13))(@browserbasehq/stagehand@1.13.0(@playwright/test@1.50.1)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.5.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@11.8.1)(cheerio@1.0.0)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.3)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.3)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.3)(playwright@1.50.1)(ws@8.18.1) + version: 0.3.24(@browserbasehq/sdk@2.3.0(encoding@0.1.13))(@browserbasehq/stagehand@1.13.0(@playwright/test@1.50.1)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.5.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@8.7.0)(cheerio@1.0.0-rc.10)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.3)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.3)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.3)(playwright@1.50.1)(ws@8.18.1) '@langchain/core': specifier: ^0.3.30 version: 0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) @@ -5508,9 +5502,6 @@ packages: bcryptjs@2.4.3: resolution: {integrity: sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==} - better-sqlite3@11.8.1: - resolution: {integrity: sha512-9BxNaBkblMjhJW8sMRZxnxVTRgbRmssZW0Oxc1MPBTfiR+WW21e2Mk4qu8CzrcZb1LwPCnFsfDEzq+SNcBU8eg==} - better-sqlite3@8.7.0: resolution: {integrity: sha512-99jZU4le+f3G6aIl6PmmV0cxUIWqKieHxsiF7G34CVFiE+/UabpYqkU0NJIkY/96mQKikHeBjtR27vFfs5JpEw==} @@ -5763,10 +5754,6 @@ packages: cheerio-select@2.1.0: resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} - cheerio@1.0.0: - resolution: {integrity: sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==} - engines: {node: '>=18.17'} - cheerio@1.0.0-rc.10: resolution: {integrity: sha512-g0J0q/O6mW8z5zxQ3A8E8J1hUgp4SMOvEoW/x84OwyHKe/Zccz83PVT4y5Crcr530FV6NgmKI1qvGTKVl9XXVw==} engines: {node: '>= 6'} @@ -6905,9 +6892,6 @@ packages: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} - encoding-sniffer@0.2.0: - resolution: {integrity: sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==} - encoding@0.1.13: resolution: {integrity: sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==} @@ -7445,9 +7429,6 @@ packages: resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} engines: {node: '>= 12.20'} - formidable@3.5.1: - resolution: {integrity: sha512-WJWKelbRHN41m5dumb0/k8TeAx7Id/y3a+Z7QfhxP/htI9Js5zYaEDtG8uMgG0vM0lOlqnmjE99/kfpOYi/0Og==} - formidable@3.5.2: resolution: {integrity: sha512-Jqc1btCy3QzRbJaICGwKcBfGWuLADRerLzDqi2NwSt/UkXLsHJw2TVResiaoBufHVHy9aSgClOHCeJsSsFLTbg==} @@ -7876,10 +7857,6 @@ packages: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} hasBin: true - hexoid@1.0.0: - resolution: {integrity: sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==} - engines: {node: '>=8'} - hexoid@2.0.0: resolution: {integrity: sha512-qlspKUK7IlSQv2o+5I7yhUd7TxlOG2Vr5LTa3ve2XSNVKAL/n/u/7KLvKmFNimomDIKvZFXWHv0T12mv7rT8Aw==} engines: {node: '>=8'} @@ -7966,9 +7943,6 @@ packages: htmlparser2@8.0.2: resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} - htmlparser2@9.1.0: - resolution: {integrity: sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==} - htmlparser@1.7.7: resolution: {integrity: sha512-zpK66ifkT0fauyFh2Mulrq4AqGTucxGtOhZ8OjkbSfcCpkqQEI8qRkY0tSQSJNAQ4HUZkgWaU4fK4EH6SVH9PQ==} engines: {node: '>=0.1.33'} @@ -9485,9 +9459,6 @@ packages: napi-build-utils@1.0.2: resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} - napi-build-utils@2.0.0: - resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} - native-promise-only@0.8.1: resolution: {integrity: sha512-zkVhZUA3y8mbz652WrL5x0fB0ehrBkulWT3TomAQ9iDtyXZvzKeEA6GPxAItBYeNYl5yngKRX612qHOhvMkDeg==} @@ -9575,10 +9546,6 @@ packages: resolution: {integrity: sha512-bLn/fU/ALVBE9wj+p4Y21ZJWYFjUXLXPi/IewyLZkx3ApxKDNBWCKdReeKOtD8dWpOdDCeMyLh6ZewzcLsG2Nw==} engines: {node: '>=10'} - node-abi@3.74.0: - resolution: {integrity: sha512-c5XK0MjkGBrQPGYG24GBADZud0NCbznxNx0ZkS+ebUTrmV1qTDxPxSL8zEAPURXSbLRWVexxmP4986BziahL5w==} - engines: {node: '>=10'} - node-addon-api@6.1.0: resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} @@ -9942,9 +9909,6 @@ packages: parse5-htmlparser2-tree-adapter@7.1.0: resolution: {integrity: sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==} - parse5-parser-stream@7.1.2: - resolution: {integrity: sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==} - parse5@6.0.1: resolution: {integrity: sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==} @@ -10357,11 +10321,6 @@ packages: engines: {node: '>=10'} hasBin: true - prebuild-install@7.1.3: - resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} - engines: {node: '>=10'} - hasBin: true - precond@0.2.3: resolution: {integrity: sha512-QCYG84SgGyGzqJ/vlMsxeXd/pgL/I94ixdNFyh1PusWmTCyVfPJjZ1K1jvHtsbfnXQs2TSkEP2fR7QiMZAnKFQ==} engines: {node: '>= 0.6'} @@ -11035,9 +10994,6 @@ packages: redux@4.2.1: resolution: {integrity: sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==} - redux@5.0.1: - resolution: {integrity: sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==} - reflect-metadata@0.1.13: resolution: {integrity: sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==} @@ -11348,11 +11304,6 @@ packages: engines: {node: '>=10'} hasBin: true - semver@7.7.1: - resolution: {integrity: sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==} - engines: {node: '>=10'} - hasBin: true - send@0.19.0: resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==} engines: {node: '>= 0.8.0'} @@ -11851,9 +11802,6 @@ packages: tar-fs@2.1.1: resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==} - tar-fs@2.1.2: - resolution: {integrity: sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==} - tar-fs@3.0.6: resolution: {integrity: sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==} @@ -12244,10 +12192,6 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici@6.21.1: - resolution: {integrity: sha512-q/1rj5D0/zayJB2FraXdaWxbhWiNKDvu8naDT2dl1yTlvJp4BLtOcp2a5BvgGNQpYYJzau7tf1WgKv3b+7mqpQ==} - engines: {node: '>=18.17'} - unicorn-magic@0.1.0: resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==} engines: {node: '>=18'} @@ -12615,14 +12559,6 @@ packages: webworkify@1.5.0: resolution: {integrity: sha512-AMcUeyXAhbACL8S2hqqdqOLqvJ8ylmIbNwUIqQujRSouf4+eUFaXbG6F1Rbu+srlJMmxQWsiU7mOJi0nMBfM1g==} - whatwg-encoding@3.1.1: - resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} - engines: {node: '>=18'} - - whatwg-mimetype@4.0.0: - resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} - engines: {node: '>=18'} - whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} @@ -13010,7 +12946,7 @@ snapshots: agentkeepalive: 4.6.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.6.7(encoding@0.1.13) transitivePeerDependencies: - encoding @@ -13695,7 +13631,7 @@ snapshots: agentkeepalive: 4.6.0 form-data-encoder: 1.7.2 formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) + node-fetch: 2.6.7(encoding@0.1.13) transitivePeerDependencies: - encoding @@ -13757,7 +13693,7 @@ snapshots: '@cocalc/primus-responder@1.0.5': dependencies: - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0 node-uuid: 1.4.8 transitivePeerDependencies: - supports-color @@ -14610,7 +14546,7 @@ snapshots: transitivePeerDependencies: - encoding - ? '@langchain/community@0.3.24(@browserbasehq/sdk@2.3.0(encoding@0.1.13))(@browserbasehq/stagehand@1.13.0(@playwright/test@1.50.1)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.5.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@11.8.1)(cheerio@1.0.0)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.3)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.3)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.3)(playwright@1.50.1)(ws@8.18.1)' + ? '@langchain/community@0.3.24(@browserbasehq/sdk@2.3.0(encoding@0.1.13))(@browserbasehq/stagehand@1.13.0(@playwright/test@1.50.1)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8))(@google-ai/generativelanguage@2.7.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@ibm-cloud/watsonx-ai@1.5.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(better-sqlite3@8.7.0)(cheerio@1.0.0-rc.10)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.3)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ibm-cloud-sdk-core@5.1.3)(ignore@7.0.3)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(pg@8.13.3)(playwright@1.50.1)(ws@8.18.1)' : dependencies: '@browserbasehq/stagehand': 1.13.0(@playwright/test@1.50.1)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))(zod@3.23.8) '@ibm-cloud/watsonx-ai': 1.5.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))) @@ -14621,7 +14557,7 @@ snapshots: flat: 5.0.2 ibm-cloud-sdk-core: 5.1.3 js-yaml: 4.1.0 - langchain: 0.3.11(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(cheerio@1.0.0)(encoding@0.1.13)(handlebars@4.7.8)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) + langchain: 0.3.11(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(cheerio@1.0.0-rc.10)(encoding@0.1.13)(handlebars@4.7.8)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) langsmith: 0.2.15(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) openai: 4.78.1(encoding@0.1.13)(zod@3.23.8) uuid: 10.0.0 @@ -14631,8 +14567,8 @@ snapshots: '@browserbasehq/sdk': 2.3.0(encoding@0.1.13) '@google-ai/generativelanguage': 2.7.0(encoding@0.1.13) '@google-cloud/storage': 7.13.0(encoding@0.1.13) - better-sqlite3: 11.8.1 - cheerio: 1.0.0 + better-sqlite3: 8.7.0 + cheerio: 1.0.0-rc.10 d3-dsv: 3.0.1 fast-xml-parser: 4.5.3 google-auth-library: 9.14.1(encoding@0.1.13) @@ -16694,7 +16630,7 @@ snapshots: axios@1.7.9(debug@4.4.0): dependencies: follow-redirects: 1.15.9(debug@4.4.0) - form-data: 4.0.0 + form-data: 4.0.2 proxy-from-env: 1.1.0 transitivePeerDependencies: - debug @@ -16867,12 +16803,6 @@ snapshots: bcryptjs@2.4.3: {} - better-sqlite3@11.8.1: - dependencies: - bindings: 1.5.0 - prebuild-install: 7.1.3 - optional: true - better-sqlite3@8.7.0: dependencies: bindings: 1.5.0 @@ -17163,21 +17093,6 @@ snapshots: domhandler: 5.0.3 domutils: 3.2.2 - cheerio@1.0.0: - dependencies: - cheerio-select: 2.1.0 - dom-serializer: 2.0.0 - domhandler: 5.0.3 - domutils: 3.2.2 - encoding-sniffer: 0.2.0 - htmlparser2: 9.1.0 - parse5: 7.2.1 - parse5-htmlparser2-tree-adapter: 7.1.0 - parse5-parser-stream: 7.1.2 - undici: 6.21.1 - whatwg-mimetype: 4.0.0 - optional: true - cheerio@1.0.0-rc.10: dependencies: cheerio-select: 1.6.0 @@ -18096,6 +18011,10 @@ snapshots: dependencies: ms: 2.1.2 + debug@4.4.0: + dependencies: + ms: 2.1.3 + debug@4.4.0(supports-color@8.1.1): dependencies: ms: 2.1.3 @@ -18455,12 +18374,6 @@ snapshots: encodeurl@2.0.0: {} - encoding-sniffer@0.2.0: - dependencies: - iconv-lite: 0.6.3 - whatwg-encoding: 3.1.1 - optional: true - encoding@0.1.13: dependencies: iconv-lite: 0.6.3 @@ -19175,12 +19088,6 @@ snapshots: node-domexception: 1.0.0 web-streams-polyfill: 4.0.0-beta.3 - formidable@3.5.1: - dependencies: - dezalgo: 1.0.4 - hexoid: 1.0.0 - once: 1.4.0 - formidable@3.5.2: dependencies: dezalgo: 1.0.4 @@ -19787,8 +19694,6 @@ snapshots: he@1.2.0: {} - hexoid@1.0.0: {} - hexoid@2.0.0: {} highlight-words-core@1.2.2: {} @@ -19905,14 +19810,6 @@ snapshots: domutils: 3.2.2 entities: 4.5.0 - htmlparser2@9.1.0: - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - domutils: 3.2.2 - entities: 4.5.0 - optional: true - htmlparser@1.7.7: {} http-deceiver@1.2.7: {} @@ -21106,7 +21003,7 @@ snapshots: lambda-cloud-node-api@1.0.1: {} - langchain@0.3.11(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(cheerio@1.0.0)(encoding@0.1.13)(handlebars@4.7.8)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)): + langchain@0.3.11(@langchain/anthropic@0.3.11(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(@langchain/google-genai@0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8))(@langchain/mistralai@0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))))(axios@1.7.7)(cheerio@1.0.0-rc.10)(encoding@0.1.13)(handlebars@4.7.8)(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)): dependencies: '@langchain/core': 0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)) '@langchain/openai': 0.3.17(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) @@ -21126,7 +21023,7 @@ snapshots: '@langchain/google-genai': 0.1.6(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8) '@langchain/mistralai': 0.2.0(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.23.8))) axios: 1.7.7 - cheerio: 1.0.0 + cheerio: 1.0.0-rc.10 handlebars: 4.7.8 transitivePeerDependencies: - encoding @@ -21727,9 +21624,6 @@ snapshots: napi-build-utils@1.0.2: {} - napi-build-utils@2.0.0: - optional: true - native-promise-only@0.8.1: {} nats.ws@1.30.1: @@ -21842,11 +21736,6 @@ snapshots: dependencies: semver: 7.6.3 - node-abi@3.74.0: - dependencies: - semver: 7.7.1 - optional: true - node-addon-api@6.1.0: {} node-addon-api@7.1.1: @@ -22301,11 +22190,6 @@ snapshots: domhandler: 5.0.3 parse5: 7.2.1 - parse5-parser-stream@7.1.2: - dependencies: - parse5: 7.2.1 - optional: true - parse5@6.0.1: {} parse5@7.2.1: @@ -22764,22 +22648,6 @@ snapshots: tar-fs: 2.1.1 tunnel-agent: 0.6.0 - prebuild-install@7.1.3: - dependencies: - detect-libc: 2.0.3 - expand-template: 2.0.3 - github-from-package: 0.0.0 - minimist: 1.2.8 - mkdirp-classic: 0.5.3 - napi-build-utils: 2.0.0 - node-abi: 3.74.0 - pump: 3.0.2 - rc: 1.2.8 - simple-get: 4.0.1 - tar-fs: 2.1.2 - tunnel-agent: 0.6.0 - optional: true - precond@0.2.3: {} predefine@0.1.3: @@ -23477,7 +23345,7 @@ snapshots: react-property@2.0.0: {} - react-redux@8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@5.0.1): + react-redux@8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@4.2.1): dependencies: '@babel/runtime': 7.25.6 '@types/hoist-non-react-statics': 3.3.1 @@ -23490,7 +23358,7 @@ snapshots: '@types/react': 18.3.10 '@types/react-dom': 18.3.0 react-dom: 18.3.1(react@18.3.1) - redux: 5.0.1 + redux: 4.2.1 react-refresh@0.14.2: {} @@ -23618,9 +23486,6 @@ snapshots: dependencies: '@babel/runtime': 7.25.6 - redux@5.0.1: - optional: true - reflect-metadata@0.1.13: {} reflect.getprototypeof@1.0.6: @@ -23982,9 +23847,6 @@ snapshots: semver@7.6.3: {} - semver@7.7.1: - optional: true - send@0.19.0: dependencies: debug: 2.6.9 @@ -24589,14 +24451,6 @@ snapshots: pump: 3.0.2 tar-stream: 2.2.0 - tar-fs@2.1.2: - dependencies: - chownr: 1.1.4 - mkdirp-classic: 0.5.3 - pump: 3.0.2 - tar-stream: 2.2.0 - optional: true - tar-fs@3.0.6: dependencies: pump: 3.0.2 @@ -24998,9 +24852,6 @@ snapshots: undici-types@5.26.5: {} - undici@6.21.1: - optional: true - unicorn-magic@0.1.0: {} unified@10.1.2: @@ -25458,7 +25309,7 @@ snapshots: dependencies: '@wwa/statvfs': 1.1.18 awaiting: 3.0.0 - debug: 4.4.0(supports-color@8.1.1) + debug: 4.4.0 port-get: 1.0.4 ws: 8.18.0 transitivePeerDependencies: @@ -25468,14 +25319,6 @@ snapshots: webworkify@1.5.0: {} - whatwg-encoding@3.1.1: - dependencies: - iconv-lite: 0.6.3 - optional: true - - whatwg-mimetype@4.0.0: - optional: true - whatwg-url@5.0.0: dependencies: tr46: 0.0.3 diff --git a/src/packages/project/nats/files/write.ts b/src/packages/project/nats/files/write.ts index 39e2e507a9..cbdaaaea9f 100644 --- a/src/packages/project/nats/files/write.ts +++ b/src/packages/project/nats/files/write.ts @@ -27,12 +27,15 @@ You can also skip step 1 if you instead set COMPUTE_SERVER_ID to something nonze import "@cocalc/project/nats/env"; // ensure nats env available import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; import { createWriteStream as fs_createWriteStream } from "fs"; +import { rename } from "fs/promises"; import { compute_server_id, project_id } from "@cocalc/project/data"; import { join } from "path"; import { createServer, close as closeWriteServer, } from "@cocalc/nats/files/write"; +import { randomId } from "@cocalc/nats/names"; +import { rimraf } from "rimraf"; async function createWriteStream(path: string) { // console.log("createWriteStream", { path }); @@ -40,7 +43,16 @@ async function createWriteStream(path: string) { path = join(process.env.HOME, path); } await ensureContainingDirectoryExists(path); - const stream = fs_createWriteStream(path); + const partial = path + `.partialupload-${randomId()}`; + const stream = fs_createWriteStream(partial); + stream.on("remove", async () => { + console.log("stream on error"); + await rimraf(partial); + }); + stream.on("rename", async () => { + console.log("stream on end"); + await rename(partial, path); + }); // TODO: path should be a temporary path to indicate that it is a partial // upload, then get moved to path when done or deleted on error. diff --git a/src/packages/project/package.json b/src/packages/project/package.json index 90cd732a4d..205ed6ec30 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -36,7 +36,6 @@ "@nats-io/kv": "3.0.0-30", "@nats-io/services": "3.0.0-25", "@nteract/messaging": "^7.0.20", - "@types/formidable": "^3.4.5", "@types/lodash": "^4.14.202", "@types/primus": "^7.3.9", "@types/uuid": "^8.3.1", @@ -50,7 +49,6 @@ "expect": "^26.6.2", "express": "^4.21.2", "express-rate-limit": "^7.4.0", - "formidable": "^3.5.2", "get-port": "^5.1.1", "googlediff": "^0.1.0", "json-stable-stringify": "^1.0.1", @@ -64,7 +62,7 @@ "prettier": "^3.0.2", "primus": "^8.0.9", "prom-client": "^13.0.0", - "serve-index": "^1.9.1", + "rimraf": "^5.0.5", "temp": "^0.9.4", "tmp": "0.0.33", "uglify-js": "^3.14.1", diff --git a/src/packages/project/servers/browser/http-server.ts b/src/packages/project/servers/browser/http-server.ts index 20a6db64a1..7a84269d31 100644 --- a/src/packages/project/servers/browser/http-server.ts +++ b/src/packages/project/servers/browser/http-server.ts @@ -10,7 +10,6 @@ this projects. It serves both HTTP and websocket connections, which should be proxied through some hub. */ -import bodyParser from "body-parser"; import compression from "compression"; import express from "express"; import { createServer } from "http"; @@ -27,10 +26,8 @@ import { getOptions } from "@cocalc/project/init-program"; import initJupyter from "@cocalc/project/jupyter/http-server"; import * as kucalc from "@cocalc/project/kucalc"; import { getLogger } from "@cocalc/project/logger"; -import initUpload from "@cocalc/project/upload"; import { once } from "@cocalc/util/async-utils"; import initRootSymbolicLink from "./root-symlink"; -import initStaticServer from "./static"; const winston = getLogger("browser-http-server"); @@ -64,12 +61,6 @@ export default async function init(): Promise { // suggested by http://expressjs.com/en/advanced/best-practice-performance.html#use-gzip-compression app.use(compression()); - // Needed for POST file to custom path, which is used for uploading files to projects. - // parse application/x-www-form-urlencoded - app.use(bodyParser.urlencoded({ extended: true })); - // parse application/json - app.use(bodyParser.json()); - winston.info("creating root symbolic link"); await initRootSymbolicLink(); @@ -93,13 +84,6 @@ export default async function init(): Promise { app.use(base, await initJupyter()); })(); - // Setup the upload POST endpoint - winston.info("initializing file upload server"); - app.use(base, initUpload()); - - winston.info("initializing static server"); - initStaticServer(app, base); - const options = getOptions(); server.listen(options.browserPort, options.hostname); await once(server, "listening"); diff --git a/src/packages/project/servers/browser/static.ts b/src/packages/project/servers/browser/static.ts deleted file mode 100644 index 1fe3c2331d..0000000000 --- a/src/packages/project/servers/browser/static.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Application, static as staticServer } from "express"; -import index from "serve-index"; -import { getLogger } from "@cocalc/project/logger"; - -export default function init(app: Application, base: string) { - const winston = getLogger("serve-static-files-to-browser"); - winston.info(`initialize with base="${base}"`); - // Setup the static raw HTTP server. This must happen after anything above, - // since it serves all URL's (so it has to be the fallback). - app.use(base, (req, res, next) => { - // this middleware function has to come before the express.static server! - // it sets the content type to octet-stream (aka "download me") if URL query ?download exists - if (req.query.download != null) { - res.setHeader("Content-Type", "application/octet-stream"); - } - // Note: we do not set no-cache since that causes major issues on Safari: - // https://github.com/sagemathinc/cocalc/issues/5120 - // By now our applications should use query params to break the cache. - res.setHeader("Cache-Control", "private, must-revalidate"); - next(); - }); - - const { HOME } = process.env; - if (HOME == null) { - throw Error("HOME env variable must be defined"); - } - winston.info(`serving up HOME="${HOME}"`); - - app.use(base, index(HOME, { hidden: true, icons: true })); - app.use(base, staticServer(HOME, { dotfiles: "allow" })); -} diff --git a/src/packages/project/upload.ts b/src/packages/project/upload.ts deleted file mode 100644 index 1c987ea63d..0000000000 --- a/src/packages/project/upload.ts +++ /dev/null @@ -1,241 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Upload form handler -*/ - -// This is a limit on the size of each *chunk* that the frontend sends, -// not the total size of the file... -const MAX_FILE_SIZE_MB = 10000; - -import { Router } from "express"; -import formidable from "formidable"; -import { promises as fs, constants as fs_constants } from "node:fs"; -import { - appendFile, - copyFile, - mkdir, - readFile, - rename, - unlink, -} from "node:fs/promises"; -import { join } from "node:path"; -import { handleCopy } from "@cocalc/sync-fs/lib/handle-api-call"; - -const { F_OK, W_OK, R_OK } = fs_constants; - -import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; -import { getLogger } from "./logger"; - -const logger = getLogger("project:upload"); - -export default function init(): Router { - logger.info("configuring the upload endpoint"); - - const router = Router(); - - router.get("/.smc/upload", function (_, res) { - logger.http("upload GET"); - return res.send("hello"); - }); - - router.post("/.smc/upload", async function (req, res): Promise { - function dbg(...m): void { - logger.http("upload POST ", ...m); - } - // See https://github.com/felixge/node-formidable; user uploaded a file - - // See http://stackoverflow.com/questions/14022353/how-to-change-upload-path-when-use-formidable-with-express-in-node-js - // Important to set maxFileSize, since the default is 200MB! - // See https://stackoverflow.com/questions/13374238/how-to-limit-upload-file-size-in-express-js - const { dest_dir } = req.query; - const compute_server_id = getComputeServerId(req); - - dbg({ dest_dir, compute_server_id }); - - if (typeof dest_dir != "string") { - res.status(500).send("query param dest_dir must be a string"); - return; - } - const { HOME } = process.env; - if (!HOME) { - throw Error("HOME env var must be set"); - } - - try { - const uploadDir = join(HOME, ".smc", "upload"); - // ensure target path exists - dbg("ensure target path exists... ", uploadDir); - await mkdir(uploadDir, { recursive: true }); - - // we check explicitly, otherwise: https://github.com/sagemathinc/cocalc/issues/7513 - dbg("check if uploadDir has read/writewrite permissions... ", uploadDir); - try { - await fs.access(uploadDir, F_OK | R_OK | W_OK); - } catch { - throw new Error("upload directory does not have write permissions"); - } - - const form = formidable({ - uploadDir, - keepExtensions: true, - maxFileSize: MAX_FILE_SIZE_MB * 1024 * 1024, - }); - - dbg("parsing form data..."); - // https://github.com/node-formidable/formidable?tab=readme-ov-file#parserequest-callback - const [fields, files] = await form.parse(req); - //dbg(`finished parsing form data. ${JSON.stringify({ fields, files })}`); - - /* Just for the sake of understanding this, this is how this looks like in the real world (formidable@3): - > files.file[0] - { - size: 80789, - filepath: '/home/hsy/p/cocalc/src/data/projects/c8787b71-a85f-437b-9d1b-29833c3a199e/asdf/asdf/8e3e4367333e45275a8d1aa03.png', - newFilename: '8e3e4367333e45275a8d1aa03.png', - mimetype: 'application/octet-stream', - mtime: '2024-04-23T09:25:53.197Z', - originalFilename: 'Screenshot from 2024-04-23 09-20-40.png' - } - - > fields - { - dzuuid: [ 'b4a26289-ddd5-42fc-bfa8-b18847a048a3' ], - dzchunkindex: [ '0' ], - dztotalfilesize: [ '80789' ], - dzchunksize: [ '8000000' ], - dztotalchunkcount: [ '1' ], - dzchunkbyteoffset: [ '0' ] - } - */ - - // Now, the strategy is to assemble the file chunk by chunk and save it with the original filename - const chunkFullPath = files.file[0]?.filepath; - const originalFn = files.file[0]?.originalFilename; - - if (chunkFullPath == null || originalFn == null) { - dbg("error parsing form data"); - throw Error("files.file[0].[filepath | originalFilename] is null"); - } else { - dbg(`uploading '${chunkFullPath}' -> '${originalFn}'`); - } - - const temp = join(uploadDir, originalFn); - const dest = join(HOME, dest_dir, originalFn); - dbg(`dest='${dest}'`); - await ensureContainingDirectoryExists(dest); - - dbg("append the next chunk onto the destination file..."); - await handle_chunk_data( - parseInt(fields.dzchunkindex), - parseInt(fields.dztotalchunkcount), - chunkFullPath, - temp, - dest, - dest_dir, - compute_server_id, - ); - - res.send("received upload:\n\n"); - } catch (err) { - dbg("upload failed ", err); - res.status(500).send(`upload failed -- ${err}`); - } - }); - return router; -} - -function getComputeServerId(req) { - try { - return parseInt(req.query.compute_server_id ?? "0"); - } catch (_) { - return 0; - } -} - -async function handle_chunk_data( - index: number, - total: number, - chunk: string, - temp: string, - dest: string, - dest_dir: string, - compute_server_id: number, -): Promise { - if (index === 0) { - logger.debug("handle_chunk_data: move chunk to the temp file"); - await moveFile(chunk, temp); - } else { - logger.debug("handle_chunk_data: append chunk to the temp file"); - const data = await readFile(chunk); - await appendFile(temp, data); - await unlink(chunk); - } - if (index === total - 1) { - logger.debug( - "handle_chunk_data: it's the last chunk, move temp to actual file.", - ); - await moveFile(temp, dest, dest_dir, compute_server_id); - } -} - -async function moveFile( - temp: string, - dest: string, - dest_dir?: string, - compute_server_id?: number, -): Promise { - try { - logger.debug("move temporary file to dest", { - temp, - dest, - }); - try { - await rename(temp, dest); - } catch (_) { - // in some cases, e.g., cocalc-docker, this happens: - // "EXDEV: cross-device link not permitted" - // so we just try again the slower way. This is slightly - // inefficient, maybe, but more robust than trying to determine - // if we are doing a cross device rename. - await copyFile(temp, dest); - } - - if (compute_server_id) { - // The final destination of this file upload is a compute server. - // We copy the temp file (temp) to the compute server, then remove - // the temp file. - // TODO: it would obviously be much more efficient to upload directly - // to the compute server without going through cocalc at all. For - // various reasons that is simply impossible in general, unfortunately. - logger.debug("moveFile: move temporary file to compute server", { - temp, - dest, - dest_dir, - compute_server_id, - }); - - // input to handleCopy must be relative to home directory, - // but temp and dest are absolute paths got by putting HOME - // in the front of them: - const { HOME } = process.env; - if (!HOME) { - throw Error("HOME env var must be set"); - } - await handleCopy({ - event: "copy_from_project_to_compute_server", - compute_server_id, - paths: [dest.slice(HOME.length + 1)], - dest: dest_dir, - }); - return; - } - } finally { - try { - await unlink(temp); - } catch (_) {} - } -} diff --git a/src/scripts/g.sh b/src/scripts/g.sh index 79310bfe67..79200af09f 100755 --- a/src/scripts/g.sh +++ b/src/scripts/g.sh @@ -8,6 +8,10 @@ export DEBUG="cocalc:*" #export DEBUG_CONSOLE="yes" unset DEBUG_CONSOLE +# Set this COCALC_DISABLE_NEXT to something nonempty to disable nextjs entirely +# which is very helpful when doing development. +# export COCALC_DISABLE_NEXT="yes" + #export COCALC_DISABLE_API_VALIDATION=yes #export NO_RSPACK_DEV_SERVER=yes From c6aadf85b2ddb8d466d767028a5e392f836353e6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 22 Feb 2025 16:31:08 +0000 Subject: [PATCH 261/281] nats upload: better error handling --- src/packages/frontend/file-upload.tsx | 33 +++++++++++++++++++----- src/packages/hub/proxy/handle-request.ts | 3 +++ src/packages/hub/servers/app/upload.ts | 26 +++++++++++++------ src/packages/nats/files/write.ts | 9 ++++--- src/packages/project/nats/files/write.ts | 2 -- 5 files changed, 53 insertions(+), 20 deletions(-) diff --git a/src/packages/frontend/file-upload.tsx b/src/packages/frontend/file-upload.tsx index c2e2008ffa..a75247b597 100644 --- a/src/packages/frontend/file-upload.tsx +++ b/src/packages/frontend/file-upload.tsx @@ -27,9 +27,10 @@ import { BASE_URL } from "@cocalc/frontend/misc"; import { MAX_BLOB_SIZE } from "@cocalc/util/db-schema/blobs"; import { defaults, is_array, merge } from "@cocalc/util/misc"; -// 3GB upload limit -- since that's the default filesystem quota -// and it should be plenty? -const MAX_FILE_SIZE_MB = 3000; +// very large upload limit -- should be plenty? +// there is no cost for ingress, and as cocalc is a data plaform +// people like to upload large data sets. +const MAX_FILE_SIZE_MB = 50 * 1000; const CHUNK_SIZE_MB = 8; @@ -61,14 +62,35 @@ given TIMEOUT_S. See also the discussion here: https://github.com/sagemathinc/cocalc-docker/issues/92 */ +// The corresponding server is in packages/hub/servers/app/upload.ts and significantly impacts +// our options! It uses formidable to capture each chunk and then rewrites it using NATS which +// reads the data and writes it to disk. const UPLOAD_OPTIONS = { maxFilesize: MAX_FILE_SIZE_MB, + // use chunking data for ALL files -- this is good because it makes our server code simpler. forceChunking: true, chunking: true, chunkSize: CHUNK_SIZE_MB * 1000 * 1000, - retryChunks: true, // might as well since it's a little more robust. - timeout: 1000 * TIMEOUT_S, // matches what cloudflare imposes on us; this + + // We do NOT support chunk retries, since our server doesn't. To support this, either our + // NATS protocol becomes much more complicated, or our server has to store at least one chunk + // in RAM before streaming it, which could potentially lead to a large amount of memory + // usage, especially with malicious users. If users really need a robust way to upload + // a *lot* of data, they should use rsync. + retryChunks: false, + + // matches what cloudflare imposes on us; this // is *per chunk*, so much larger uploads should still work. + // This is per chunk: + timeout: 1000 * TIMEOUT_S, + + // this is the default, but also I wrote the server (see packages/hub/servers/app/upload.ts) and + // it doesn't support parallel chunks, which would use a lot more RAM on the server. We might + // consider this later... + parallelChunkUploads: false, + + thumbnailWidth: 240, + thumbnailheight: 240, }; const DROPSTYLE = { @@ -91,7 +113,6 @@ function Header({ close_preview }: { close_preview?: Function }) { Drag and drop files from your computer {close_preview && (