Introduce Postgres DB Adapter

README.md

@@ -82,6 +82,7 @@ Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all`
 | :4204 | `root (/)` drafts realm                                   | ✅                  | 🚫                   |
 | :4205 | qunit server mounting realms in iframes for testing       | ✅                  | 🚫                   |
 | :5001 | Mail user interface for viewing emails sent to local SMTP | ✅                  | 🚫                   |
+| :5435 | Postgres DB                                               | ✅                  | 🚫                   |
 | :8008 | Matrix synapse server                                     | ✅                  | 🚫                   |
 
 #### Using `start:development`

packages/host/tests/unit/indexer-test.ts

@@ -9,7 +9,7 @@ import {
 } from '@cardstack/runtime-common';
 
 import ENV from '@cardstack/host/config/environment';
-import SQLiteAdapter from '@cardstack/host/lib/SQLiteAdapter';
+import SQLiteAdapter from '@cardstack/host/lib/sqlite-adapter';
 
 import { testRealmURL, setupIndex } from '../helpers';
 

packages/host/tests/unit/query-test.ts

@@ -12,8 +12,8 @@ import {
 } from '@cardstack/runtime-common';
 
 import ENV from '@cardstack/host/config/environment';
-import SQLiteAdapter from '@cardstack/host/lib/SQLiteAdapter';
 import { shimExternals } from '@cardstack/host/lib/externals';
+import SQLiteAdapter from '@cardstack/host/lib/sqlite-adapter';
 
 import { CardDef } from 'https://cardstack.com/base/card-api';
 

packages/host/tests/unit/sqlite-test.ts

@@ -1,6 +1,6 @@
 import { module, test } from 'qunit';
 
-import SQLiteAdapter from '@cardstack/host/lib/SQLiteAdapter';
+import SQLiteAdapter from '@cardstack/host/lib/sqlite-adapter';
 
 module('Unit | sqlite | SQLiteAdapter', function () {
   test('run a sqlite db using the SQLiteAdapter', async function (assert) {

packages/realm-server/migrations/.eslintrc.js

@@ -0,0 +1,17 @@
+'use strict';
+
+module.exports = {
+  root: true,
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    sourceType: 'script',
+  },
+  env: {
+    browser: false,
+    node: true,
+  },
+  extends: ['plugin:n/recommended'],
+  rules: {
+    camelcase: 'off',
+  },
+};

packages/realm-server/migrations/1712771547705_initial.js

@@ -0,0 +1,62 @@
+exports.up = (pgm) => {
+  pgm.createTable('indexed_cards', {
+    card_url: { type: 'varchar', notNull: true },
+    realm_version: { type: 'varchar', notNull: true },
+    realm_url: { type: 'varchar', notNull: true },
+    pristine_doc: 'jsonb',
+    search_doc: 'jsonb',
+    error_doc: 'jsonb',
+    deps: 'jsonb',
+    types: 'jsonb',
+    embedded_html: 'varchar',
+    isolated_html: 'varchar',
+    indexed_at: 'integer',
+    is_deleted: 'boolean',
+  });
+  pgm.sql('ALTER TABLE indexed_cards SET UNLOGGED');
+  pgm.addConstraint('indexed_cards', 'indexed_cards_pkey', {
+    primaryKey: ['card_url', 'realm_version'],
+  });
+  pgm.createIndex('indexed_cards', ['realm_version']);
+  pgm.createIndex('indexed_cards', ['realm_url']);
+
+  pgm.createTable('realm_versions', {
+    realm_url: { type: 'varchar', notNull: true },
+    current_version: { type: 'integer', notNull: true },
+  });
+
+  pgm.sql('ALTER TABLE realm_versions SET UNLOGGED');
+  pgm.addConstraint('realm_versions', 'realm_versions_pkey', {
+    primaryKey: ['realm_url'],
+  });
+  pgm.createIndex('realm_versions', ['current_version']);
+
+  pgm.createType('job_statuses', ['unfulfilled', 'resolved', 'rejected']);
+  pgm.createTable('jobs', {
+    id: 'id', // shorthand for primary key that is an auto incremented id
+    category: {
+      type: 'varchar',
+      notNull: true,
+    },
+    args: 'jsonb',
+    status: {
+      type: 'job_statuses',
+      default: 'unfulfilled',
+      notNull: true,
+    },
+    created_at: {
+      type: 'timestamp',
+      notNull: true,
+      default: pgm.func('current_timestamp'),
+    },
+    finished_at: {
+      type: 'timestamp',
+    },
+    queue: {
+      type: 'varchar',
+      notNull: true,
+    },
+    result: 'jsonb',
+  });
+  pgm.sql('ALTER TABLE jobs SET UNLOGGED');
+};

packages/realm-server/package.json

@@ -21,6 +21,8 @@
     "@types/lodash": "^4.14.182",
     "@types/mime-types": "^2.1.1",
     "@types/node": "^18.18.5",
+    "@types/node-pg-migrate": "^2.3.1",
+    "@types/pg": "^8.11.5",
     "@types/qs": "^6.9.14",
     "@types/qunit": "^2.11.3",
     "@types/sane": "^2.0.1",
@@ -46,7 +48,9 @@
     "lodash": "^4.17.21",
     "loglevel": "^1.8.1",
     "mime-types": "^2.1.35",
+    "node-pg-migrate": "^6.2.2",
     "npm-run-all": "^4.1.5",
+    "pg": "^8.11.5",
     "prettier": "^2.8.4",
     "prettier-plugin-ember-template-tag": "^1.1.0",
     "qs": "^6.10.5",
@@ -63,11 +67,13 @@
     "yargs": "^17.5.1"
   },
   "scripts": {
-    "test": "NODE_NO_WARNINGS=1 qunit --require ts-node/register/transpile-only tests/index.ts",
+    "test": "./scripts/remove-test-dbs.sh && NODE_NO_WARNINGS=1 PGPORT=5435 qunit --require ts-node/register/transpile-only tests/index.ts",
     "test:dom": "cd ../matrix && pnpm register-test-user && cd ../realm-server && start-server-and-test 'pnpm run wait' 'http://127.0.0.1:4205' 'testem ci -f ./testem.js'",
     "start:test-container": "http-server ./dom-tests -p 4205 --silent",
     "start:matrix": "cd ../matrix && pnpm assert-synapse-running",
     "start:smtp": "cd ../matrix && pnpm assert-smtp-running",
+    "start:pg": "./scripts/start-pg.sh",
+    "stop:pg": "./scripts/stop-pg.sh",
     "test:wait-for-servers": "NODE_NO_WARNINGS=1 start-server-and-test 'pnpm run wait' 'http-get://localhost:4201/base/fields/boolean-field?acceptHeader=application%2Fvnd.card%2Bjson' 'pnpm run wait' 'http-get://localhost:4202/node-test/person-1?acceptHeader=application%2Fvnd.card%2Bjson|http://localhost:8008|http://localhost:5001' 'test'",
     "setup:base-in-deployment": "rm -rf /persistent/base && cp -R ../base /persistent",
     "setup:drafts-in-deployment": "mkdir -p /persistent/drafts && cp --verbose --update --recursive ../drafts-realm/. /persistent/drafts/",
@@ -88,7 +94,8 @@
     "lint:fix": "concurrently \"pnpm:lint:*:fix\" --names \"fix:\"",
     "lint:js": "eslint . --cache",
     "lint:js:fix": "eslint . --fix",
-    "lint:glint": "glint"
+    "lint:glint": "glint",
+    "migrate": "node-pg-migrate"
   },
   "volta": {
     "extends": "../../package.json"

packages/realm-server/pg-adapter.ts

@@ -0,0 +1,100 @@
+import {
+  type DBAdapter,
+  type PgPrimitive,
+  type ExecuteOptions,
+} from '@cardstack/runtime-common';
+import migrate from 'node-pg-migrate';
+import { join } from 'path';
+import { Pool, Client } from 'pg';
+
+import postgresConfig from './pg-config';
+
+function config() {
+  return postgresConfig({
+    database: 'boxel',
+  });
+}
+
+export default class PgAdapter implements DBAdapter {
+  private pool: Pool;
+
+  constructor() {
+    let { user, host, database, password, port } = config();
+    console.log(`connecting to DB ${user}@${host}:${port}/${database}`);
+    this.pool = new Pool({
+      user,
+      host,
+      database,
+      password,
+      port,
+    });
+  }
+
+  async startClient() {
+    await this.migrateDb();
+  }
+
+  async close() {
+    if (this.pool) {
+      await this.pool.end();
+    }
+  }
+
+  async execute(
+    sql: string,
+    opts?: ExecuteOptions,
+  ): Promise<Record<string, PgPrimitive>[]> {
+    let client = await this.pool.connect();
+    try {
+      let { rows } = await client.query({ text: sql, values: opts?.bind });
+      return rows;
+    } catch (e: any) {
+      console.error(
+        `Error executing SQL ${e.result.message}:\n${sql}${
+          opts?.bind ? ' with bindings: ' + JSON.stringify(opts?.bind) : ''
+        }`,
+        e,
+      );
+      throw e;
+    } finally {
+      client.release();
+    }
+  }
+
+  private async migrateDb() {
+    const config = postgresConfig();
+    let client = new Client(
+      Object.assign({}, config, { database: 'postgres' }),
+    );
+    try {
+      await client.connect();
+      let response = await client.query(
+        `select count(*)=1 as has_database from pg_database where datname=$1`,
+        [config.database],
+      );
+      if (!response.rows[0].has_database) {
+        await client.query(`create database ${config.database}`);
+      }
+    } finally {
+      client.end();
+    }
+
+    await migrate({
+      direction: 'up',
+      migrationsTable: 'migrations',
+      singleTransaction: true,
+      checkOrder: false,
+      databaseUrl: {
+        user: config.user,
+        host: config.host,
+        database: config.database,
+        password: config.password,
+        port: config.port,
+      },
+      count: Infinity,
+      dir: join(__dirname, 'migrations'),
+      ignorePattern: '.*\\.eslintrc\\.js',
+      log: (...args) => console.log(...args),
+    });
+  }
+}

packages/realm-server/pg-config.ts

@@ -0,0 +1,11 @@
+import { type ClientConfig } from 'pg';
+
+export default function postgresConfig(defaultConfig: ClientConfig = {}) {
+  return Object.assign({}, defaultConfig, {
+    host: process.env.PGHOST || 'localhost',
+    port: process.env.PGPORT || '5432',
+    user: process.env.PGUSER || 'postgres',
+    password: process.env.PGPASSWORD || undefined,
+    database: process.env.PGDATABASE || 'postgres',
+  });
+}

packages/realm-server/scripts/remove-test-dbs.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+databases=$(docker exec boxel-pg psql -U postgres -w -lqt | cut -d \| -f 1 | grep -E 'test_db_' | tr -d ' ')
+
+for db in $databases; do
+  docker exec boxel-pg dropdb -U postgres -w $db
+done

packages/realm-server/scripts/start-all.sh

@@ -1,7 +1,7 @@
 #! /bin/sh
 
 NODE_NO_WARNINGS=1 start-server-and-test \
-  'run-p start:matrix start:smtp start:development start:base:root' \
+  'run-p start:pg start:matrix start:smtp start:development start:base:root' \
   'http-get://localhost:4201/base/fields/boolean-field?acceptHeader=application%2Fvnd.card%2Bjson|http-get://localhost:4203/fields/boolean-field?acceptHeader=application%2Fvnd.card%2Bjson|http-get://localhost:4201/drafts/index?acceptHeader=application%2Fvnd.card%2Bjson|http://localhost:8008|http://localhost:5001' \
   'run-p start:test-realms start:test-container' \
   'http-get://localhost:4202/node-test/person-1?acceptHeader=application%2Fvnd.card%2Bjson|http-get://127.0.0.1:4205' \

packages/realm-server/scripts/start-development.sh

@@ -1,6 +1,11 @@
 #! /bin/sh
 pnpm setup:base-assets
-NODE_ENV=development NODE_NO_WARNINGS=1 LOG_LEVELS='*=info' REALM_SECRET_SEED="shhh! it's a secret"  ts-node \
+NODE_ENV=development \
+  NODE_NO_WARNINGS=1 \
+  LOG_LEVELS='*=info' \
+  REALM_SECRET_SEED="shhh! it's a secret" \
+  PGPORT="5435" \
+  ts-node \
   --transpileOnly main \
   --port=4201 \
   \

packages/realm-server/scripts/start-pg.sh

@@ -0,0 +1,8 @@
+#! /bin/sh
+if [ -z "$(docker ps -f name=boxel-pg --all --format '{{.Names}}')" ]; then
+  # running postgres on port 5435 so it doesn't collide with native postgres
+  # that may be running on your system
+  docker run --name boxel-pg -e POSTGRES_HOST_AUTH_METHOD=trust -p 5435:5432 -d postgres >/dev/null
+else
+  docker start boxel-pg >/dev/null
+fi

packages/realm-server/scripts/start-test-realms.sh

@@ -1,6 +1,10 @@
 #! /bin/sh
 
-NODE_ENV=test NODE_NO_WARNINGS=1 REALM_SECRET_SEED="shhh! it's a secret" ts-node \
+NODE_ENV=test \
+  NODE_NO_WARNINGS=1 \
+  REALM_SECRET_SEED="shhh! it's a secret" \
+  PGPORT="5435" \
+  ts-node \
   --transpileOnly main \
   --port=4202 \
   \

packages/realm-server/scripts/stop-pg.sh

@@ -0,0 +1,2 @@
+#! /bin/sh
+docker stop boxel-pg >/dev/null

packages/realm-server/tests/helpers/index.ts

@@ -28,6 +28,10 @@ let basePath = resolve(join(__dirname, '..', '..', '..', 'base'));
 let manager = new RunnerOptionsManager();
 let getRunner: IndexRunner | undefined;
 
+export async function prepareTestDB() {
+  process.env.PGDATABASE = `test_db_${Math.floor(10000000 * Math.random())}`;
+}
+
 export async function createRealm(
   loader: Loader,
   dir: string,

packages/realm-server/tests/index.ts

@@ -5,3 +5,4 @@ import './indexing-test';
 import './module-syntax-test';
 import './permissions/permission-checker-test';
 import './auth-client-test';
+import './pg-test';