Merge pull request #99 from supabase-community/feat/db-sharing

feat: database live share

Author: gregnr

.vscode/settings.json

@@ -1,5 +1,20 @@
 {
   "deno.enablePaths": ["supabase/functions"],
   "deno.lint": true,
-  "deno.unstable": true
+  "deno.unstable": true,
+  "[javascript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[json]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[jsonc]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[typescript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[typescriptreact]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  }
 }

apps/browser-proxy/.env.example

@@ -0,0 +1,11 @@
+AWS_ACCESS_KEY_ID="<aws-access-key-id>"
+AWS_ENDPOINT_URL_S3="<aws-endpoint-url-s3>"
+AWS_S3_BUCKET=storage
+AWS_SECRET_ACCESS_KEY="<aws-secret-access-key>"
+AWS_REGION=us-east-1
+LOGFLARE_SOURCE_URL="<logflare-source-url>"
+# enable PROXY protocol support
+#PROXIED=true
+SUPABASE_URL="<supabase-url>"
+SUPABASE_ANON_KEY="<supabase-anon-key>"
+WILDCARD_DOMAIN=browser.staging.db.build

apps/browser-proxy/.gitignore

@@ -0,0 +1 @@
+tls

apps/browser-proxy/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:22-alpine
+
+WORKDIR /app
+
+COPY --link package.json ./
+COPY --link src/ ./src/
+
+RUN npm install
+
+EXPOSE 443
+EXPOSE 5432
+
+CMD ["node", "--experimental-strip-types", "src/index.ts"]

apps/browser-proxy/README.md

@@ -0,0 +1,33 @@
+# Browser Proxy
+
+This app is a proxy that sits between the browser and a PostgreSQL client.
+
+It is using a WebSocket server and a TCP server to make the communication between the PGlite instance in the browser and a standard PostgreSQL client possible.
+
+## Development
+
+Copy the `.env.example` file to `.env` and set the correct environment variables.
+
+Install dependencies:
+
+```sh
+npm install
+```
+
+Start the proxy in development mode:
+
+```sh
+npm run dev
+```
+
+## Deployment
+
+Create a new app on Fly.io, for example `database-build-browser-proxy`.
+
+Fill the app's secrets with the correct environment variables based on the `.env.example` file.
+
+Deploy the app:
+
+```sh
+fly deploy --app database-build-browser-proxy
+```

apps/browser-proxy/fly.toml

@@ -0,0 +1,23 @@
+primary_region = 'iad'
+
+[[services]]
+internal_port = 5432
+protocol = "tcp"
+[[services.ports]]
+handlers = ["proxy_proto"]
+port = 5432
+
+[[services]]
+internal_port = 443
+protocol = "tcp"
+[[services.ports]]
+port = 443
+
+[[restart]]
+policy = "always"
+retries = 10
+
+[[vm]]
+memory = '512mb'
+cpu_kind = 'shared'
+cpus = 1

apps/browser-proxy/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@database.build/browser-proxy",
+  "type": "module",
+  "scripts": {
+    "start": "node --env-file=.env --experimental-strip-types src/index.ts",
+    "dev": "node --watch --env-file=.env --experimental-strip-types src/index.ts",
+    "type-check": "tsc"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.645.0",
+    "@supabase/supabase-js": "^2.45.4",
+    "debug": "^4.3.7",
+    "expiry-map": "^2.0.0",
+    "findhit-proxywrap": "^0.3.13",
+    "nanoid": "^5.0.7",
+    "p-memoize": "^7.1.1",
+    "pg-gateway": "^0.3.0-beta.3",
+    "ws": "^8.18.0"
+  },
+  "devDependencies": {
+    "@total-typescript/tsconfig": "^1.0.4",
+    "@types/debug": "^4.1.12",
+    "@types/node": "^22.5.4",
+    "typescript": "^5.5.4"
+  }
+}

apps/browser-proxy/src/connection-manager.ts

@@ -0,0 +1,58 @@
+import type { PostgresConnection } from 'pg-gateway'
+import type { WebSocket } from 'ws'
+
+type DatabaseId = string
+type ConnectionId = string
+
+class ConnectionManager {
+  private socketsByDatabase: Map<DatabaseId, ConnectionId> = new Map()
+  private sockets: Map<ConnectionId, PostgresConnection> = new Map()
+  private websockets: Map<DatabaseId, WebSocket> = new Map()
+
+  constructor() {}
+
+  public hasSocketForDatabase(databaseId: DatabaseId) {
+    return this.socketsByDatabase.has(databaseId)
+  }
+
+  public getSocket(connectionId: ConnectionId) {
+    return this.sockets.get(connectionId)
+  }
+
+  public getSocketForDatabase(databaseId: DatabaseId) {
+    const connectionId = this.socketsByDatabase.get(databaseId)
+    return connectionId ? this.sockets.get(connectionId) : undefined
+  }
+
+  public setSocket(databaseId: DatabaseId, connectionId: ConnectionId, socket: PostgresConnection) {
+    this.sockets.set(connectionId, socket)
+    this.socketsByDatabase.set(databaseId, connectionId)
+  }
+
+  public deleteSocketForDatabase(databaseId: DatabaseId) {
+    const connectionId = this.socketsByDatabase.get(databaseId)
+    this.socketsByDatabase.delete(databaseId)
+    if (connectionId) {
+      this.sockets.delete(connectionId)
+    }
+  }
+
+  public hasWebsocket(databaseId: DatabaseId) {
+    return this.websockets.has(databaseId)
+  }
+
+  public getWebsocket(databaseId: DatabaseId) {
+    return this.websockets.get(databaseId)
+  }
+
+  public setWebsocket(databaseId: DatabaseId, websocket: WebSocket) {
+    this.websockets.set(databaseId, websocket)
+  }
+
+  public deleteWebsocket(databaseId: DatabaseId) {
+    this.websockets.delete(databaseId)
+    this.deleteSocketForDatabase(databaseId)
+  }
+}
+
+export const connectionManager = new ConnectionManager()

apps/browser-proxy/src/create-message.ts

@@ -0,0 +1,55 @@
+export function createStartupMessage(
+  user: string,
+  database: string,
+  additionalParams: Record<string, string> = {}
+): Uint8Array {
+  const encoder = new TextEncoder()
+
+  // Protocol version number (3.0)
+  const protocolVersion = 196608
+
+  // Combine required and additional parameters
+  const params = {
+    user,
+    database,
+    ...additionalParams,
+  }
+
+  // Calculate total message length
+  let messageLength = 4 // Protocol version
+  for (const [key, value] of Object.entries(params)) {
+    messageLength += key.length + 1 + value.length + 1
+  }
+  messageLength += 1 // Null terminator
+
+  const uint8Array = new Uint8Array(4 + messageLength)
+  const view = new DataView(uint8Array.buffer)
+
+  let offset = 0
+  view.setInt32(offset, messageLength + 4, false) // Total message length (including itself)
+  offset += 4
+  view.setInt32(offset, protocolVersion, false) // Protocol version number
+  offset += 4
+
+  // Write key-value pairs
+  for (const [key, value] of Object.entries(params)) {
+    uint8Array.set(encoder.encode(key), offset)
+    offset += key.length
+    uint8Array.set([0], offset++) // Null terminator for key
+    uint8Array.set(encoder.encode(value), offset)
+    offset += value.length
+    uint8Array.set([0], offset++) // Null terminator for value
+  }
+
+  uint8Array.set([0], offset) // Final null terminator
+
+  return uint8Array
+}
+
+export function createTerminateMessage(): Uint8Array {
+  const uint8Array = new Uint8Array(5)
+  const view = new DataView(uint8Array.buffer)
+  view.setUint8(0, 'X'.charCodeAt(0))
+  view.setUint32(1, 4, false)
+  return uint8Array
+}

apps/browser-proxy/src/debug.ts

@@ -0,0 +1,5 @@
+import createDebug from 'debug'
+
+createDebug.formatters.e = (fn) => fn()
+
+export const debug = createDebug('browser-proxy')

apps/browser-proxy/src/extract-ip.ts

@@ -0,0 +1,16 @@
+import { isIPv4 } from 'node:net'
+
+export function extractIP(address: string): string {
+  if (isIPv4(address)) {
+    return address
+  }
+
+  // Check if it's an IPv4-mapped IPv6 address
+  const ipv4 = address.match(/::ffff:(\d+\.\d+\.\d+\.\d+)/)
+  if (ipv4) {
+    return ipv4[1]!
+  }
+
+  // We assume it's an IPv6 address
+  return address
+}

apps/browser-proxy/src/findhit-proxywrap.types.d.ts

@@ -0,0 +1,6 @@
+module 'findhit-proxywrap' {
+  const module = {
+    proxy: (net: typeof import('node:net')) => typeof net,
+  }
+  export default module
+}

apps/browser-proxy/src/index.ts

@@ -0,0 +1,37 @@
+import { httpsServer } from './websocket-server.ts'
+import { tcpServer } from './tcp-server.ts'
+
+process.on('unhandledRejection', (reason, promise) => {
+  console.error({ location: 'unhandledRejection', reason, promise })
+})
+
+process.on('uncaughtException', (error) => {
+  console.error({ location: 'uncaughtException', error })
+})
+
+httpsServer.listen(443, () => {
+  console.log('websocket server listening on port 443')
+})
+
+tcpServer.listen(5432, () => {
+  console.log('tcp server listening on port 5432')
+})
+
+const shutdown = async () => {
+  await Promise.allSettled([
+    new Promise<void>((res) =>
+      httpsServer.close(() => {
+        res()
+      })
+    ),
+    new Promise<void>((res) =>
+      tcpServer.close(() => {
+        res()
+      })
+    ),
+  ])
+  process.exit(0)
+}
+
+process.on('SIGTERM', shutdown)
+process.on('SIGINT', shutdown)

apps/browser-proxy/src/pg-dump-middleware/constants.ts

@@ -0,0 +1,2 @@
+export const VECTOR_OID = 99999
+export const FIRST_NORMAL_OID = 16384