Integrate postgres into realm server

.github/workflows/ci.yaml

@@ -327,11 +327,11 @@ jobs:
             https://api.github.com/repos/$REPOSITORY/statuses/$HEAD_SHA \
             -d '{"context":"Matrix Playwright tests report","description":"","target_url":"'"$PLAYWRIGHT_REPORT_URL"'","state":"success"}'
 
-  realm-server-test:
-    name: Realm Server Tests
+  realm-server-in-memory-index-test:
+    name: Realm Server Tests - in-memory index
     runs-on: ubuntu-latest
     concurrency:
-      group: realm-server-test-${{ github.head_ref || github.run_id }}
+      group: realm-server-in-memory-index-test-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     steps:
       - uses: actions/checkout@v3
@@ -357,6 +357,36 @@ jobs:
         run: pnpm test:dom
         working-directory: packages/realm-server
 
+  realm-server-db-index-test:
+    name: Realm Server Tests - db index
+    runs-on: ubuntu-latest
+    concurrency:
+      group: realm-server-db-index-test-${{ github.head_ref || github.run_id }}
+      cancel-in-progress: true
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/init
+      - name: Build boxel-ui
+        run: pnpm build
+        working-directory: packages/boxel-ui/addon
+      - name: Build host dist/ for fastboot
+        run: pnpm build
+        env:
+          NODE_OPTIONS: --max_old_space_size=4096
+        working-directory: packages/host
+      - name: Start realm servers
+        run: PG_INDEXER=true pnpm start:all &
+        working-directory: packages/realm-server
+      - name: create realm users
+        run: pnpm register-realm-users
+        working-directory: packages/matrix
+      - name: realm server test suite
+        run: PG_INDEXER=true pnpm test:wait-for-servers
+        working-directory: packages/realm-server
+      - name: realm server DOM tests
+        run: PG_INDEXER=true pnpm test:dom
+        working-directory: packages/realm-server
+
   change-check:
     name: Check which packages changed
     if: github.ref == 'refs/heads/main'
@@ -396,7 +426,7 @@ jobs:
       - boxel-ui-test
       # don't forget to change this after we remove the feature flag
       - host-test-in-memory-index
-      - realm-server-test
+      - realm-server-in-memory-index-test
     uses: ./.github/workflows/manual-deploy.yml
     secrets: inherit
     with:

README.md

@@ -115,12 +115,17 @@ Boxel uses a Postgres database. In development, the Postgres database runs withi
 
 When running tests we isolate the database between each test run by actually creating a new database for each test with a random database name (e.g. `test_db_1234567`). The test databases are dropped before the beginning of each test run.
 
-If you wish to drop the development database you can execute:
+If you wish to drop the development databases you can execute:
 ```
-pnpm drop-db
+pnpm drop-all-dbs
 ```
 
-You can then run `pnpm migrate up` or start the realm server to create the database again.
+You can then run `PGDATABASE=boxel_dev pnpm migrate up` (with `PGDATABASE` set accordingly) or just start the realm server (`PG_INDEXER=true pnpm start:all`) to create the database again.
+
+To interact with your local database directly you can use psql:
+```
+psql -h localhost -p 5435 -U postgres
+```
 
 #### DB Migrations
 When the realm server starts up it will automatically run DB migrations that live in the `packages/realm-server/migrations` folder. As part of development you may wish to run migrations manually as well as to create a new migration.

packages/host/app/components/card-prerender.gts

@@ -150,6 +150,7 @@ export default class CardPrerender extends Component {
   private getRunnerParams(): {
     reader: Reader;
     entrySetter: EntrySetter;
+    // TODO make this required after feature flag removed
     indexer?: Indexer;
   } {
     let self = this;
@@ -172,6 +173,7 @@ export default class CardPrerender extends Component {
       return {
         reader: getRunnerOpts(optsId).reader,
         entrySetter: getRunnerOpts(optsId).entrySetter,
+        indexer: getRunnerOpts(optsId).indexer,
       };
     } else {
       return {

packages/host/app/lib/current-run.ts

@@ -115,9 +115,6 @@ export class CurrentRun {
     entrySetter: EntrySetter;
     renderCard: RenderCard;
   }) {
-    if (isDbIndexerEnabled()) {
-      log.info(`current-run is using db index`);
-    }
     this.#indexer = indexer;
     this.#realmPaths = new RealmPaths(realmURL);
     this.#reader = reader;

packages/host/tests/acceptance/code-submode/editor-test.ts

@@ -19,7 +19,6 @@ import type EnvironmentService from '@cardstack/host/services/environment-servic
 import type MonacoService from '@cardstack/host/services/monaco-service';
 
 import {
-  percySnapshot,
   setupLocalIndexing,
   setupServerSentEvents,
   setupOnSave,
@@ -28,7 +27,6 @@ import {
   setMonacoContent,
   setupAcceptanceTestRealm,
   visitOperatorMode,
-  waitForSyntaxHighlighting,
   waitForCodeEditor,
   type TestContextWithSSE,
   type TestContextWithSave,
@@ -332,8 +330,12 @@ module('Acceptance | code submode | editor tests', function (hooks) {
         },
       },
     });
-    await waitForSyntaxHighlighting('"Pet"', 'rgb(4, 81, 165)');
-    await percySnapshot(assert);
+
+    // TODO we often timeout waiting for syntax highlighting, so i'm commenting
+    // out this assertion and creating a ticket to research this: CS-6770
+
+    // await waitForSyntaxHighlighting('"Pet"', 'rgb(4, 81, 165)');
+    // await percySnapshot(assert);
   });
 
   test<

packages/host/tests/integration/components/operator-mode-test.gts

@@ -3654,7 +3654,9 @@ module('Integration | operator-mode', function (hooks) {
     assert
       .dom('[data-test-card-url-bar-error]')
       .containsText('This resource does not exist');
-    await percySnapshot(assert);
+    // Percy is failing to capture this snapshot for some
+    // reason. creating issue for this CS-6780
+    // await percySnapshot(assert);
 
     await fillIn('[data-test-card-url-bar-input]', `Wrong URL`);
     await triggerKeyEvent(

packages/realm-server/Dockerfile

@@ -21,4 +21,5 @@ RUN CI=1 pnpm install -r --offline
 
 EXPOSE 3000
 
+# TODO need to set the PG ENV vars to connect to the DB in the command below
 CMD pnpm --filter "./packages/realm-server" $realm_server_script

packages/realm-server/main.ts

@@ -1,5 +1,10 @@
 import './setup-logger'; // This should be first
-import { Realm, VirtualNetwork, logger } from '@cardstack/runtime-common';
+import {
+  Realm,
+  Worker,
+  VirtualNetwork,
+  logger,
+} from '@cardstack/runtime-common';
 import { NodeAdapter } from './node-realm';
 import yargs from 'yargs';
 import { RealmServer } from './server';
@@ -11,6 +16,8 @@ import { shimExternals } from './lib/externals';
 import { type RealmPermissions as RealmPermissionsInterface } from '@cardstack/runtime-common/realm';
 import * as Sentry from '@sentry/node';
 import { setErrorReporter } from '@cardstack/runtime-common/realm';
+import PgAdapter from './pg-adapter';
+import PgQueue from './pg-queue';
 
 import fs from 'fs';
 
@@ -155,6 +162,14 @@ if (distURL) {
 
 (async () => {
   let realms: Realm[] = [];
+  let dbAdapter: PgAdapter | undefined;
+  let queue: PgQueue | undefined;
+  if (process.env.PG_INDEXER) {
+    dbAdapter = new PgAdapter();
+    queue = new PgQueue(dbAdapter);
+    await dbAdapter.startClient();
+  }
+
   for (let [i, path] of paths.entries()) {
     let url = hrefs[i][0];
     let manager = new RunnerOptionsManager();
@@ -179,11 +194,11 @@ if (distURL) {
     );
 
     let realmPermissions = getRealmPermissions(url);
-
+    let realmAdapter = new NodeAdapter(resolve(String(path)));
     let realm = new Realm(
       {
         url,
-        adapter: new NodeAdapter(resolve(String(path))),
+        adapter: realmAdapter,
         indexRunner: getRunner,
         runnerOptsMgr: manager,
         getIndexHTML: async () =>
@@ -192,6 +207,23 @@ if (distURL) {
         realmSecretSeed: REALM_SECRET_SEED,
         permissions: realmPermissions.users,
         virtualNetwork,
+        // TODO remove this guard after the feature flag is removed
+        ...(dbAdapter && queue ? { dbAdapter, queue } : {}),
+        onIndexer: async (indexer) => {
+          // TODO remove this guard after the feature flag is removed
+          if (queue) {
+            let worker = new Worker({
+              realmURL: new URL(url),
+              indexer,
+              queue,
+              realmAdapter,
+              runnerOptsManager: manager,
+              loader: virtualNetwork.createLoader(),
+              indexRunner: getRunner,
+            });
+            await worker.run();
+          }
+        },
       },
       {
         deferStartUp: true,

packages/realm-server/package.json

@@ -80,7 +80,7 @@
     "setup:drafts-in-deployment": "mkdir -p /persistent/drafts && cp --verbose --update --recursive ../drafts-realm/. /persistent/drafts/",
     "setup:published-in-deployment": "mkdir -p /persistent/published && cp --verbose --update --recursive ../published-realm/. /persistent/published/",
     "setup:base-assets": "ts-node --transpileOnly ./scripts/setup-base.ts",
-    "start": "NODE_NO_WARNINGS=1 ts-node --transpileOnly main",
+    "start": "PGPORT=5435 NODE_NO_WARNINGS=1 ts-node --transpileOnly main",
     "start:base": "./scripts/start-base.sh",
     "start:test-realms": "./scripts/start-test-realms.sh",
     "start:base:root": "./scripts/start-base-root.sh",
@@ -98,7 +98,8 @@
     "lint:glint": "glint",
     "migrate": "PGDATABASE=boxel ./scripts/ensure-db-exists.sh && PGPORT=5435 PGDATABASE=boxel PGUSER=postgres node-pg-migrate",
     "make-schema": "./scripts/schema-dump.sh",
-    "drop-db": "docker exec boxel-pg dropdb -U postgres -w boxel"
+    "drop-db": "docker exec boxel-pg dropdb -U postgres -w",
+    "drop-all-dbs": "./scripts/drop-all-dbs.sh"
   },
   "volta": {
     "extends": "../../package.json"

packages/realm-server/pg-queue.ts

@@ -112,13 +112,14 @@ export default class PgQueue implements Queue {
   #isDestroyed = false;
 
   private pollInterval = 10000;
-  private pgClient = new PgAdapter();
   private handlers: Map<string, Function> = new Map();
   private notifiers: Map<number, Deferred<any>> = new Map();
 
   private jobRunner: WorkLoop | undefined;
   private notificationRunner: WorkLoop | undefined;
 
+  constructor(private pgClient: PgAdapter) {}
+
   private async query(expression: Expression) {
     return await query(this.pgClient, expression);
   }

packages/realm-server/scripts/drop-all-dbs.sh

@@ -0,0 +1,6 @@
+#! /bin/sh
+
+pnpm run drop-db boxel_dev
+pnpm run drop-db boxel_test
+pnpm run drop-db boxel_dev_base
+pnpm run drop-db boxel_test_base_root

packages/realm-server/scripts/start-base-root.sh

@@ -1,6 +1,15 @@
 #! /bin/sh
+SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+. "$SCRIPTS_DIR/wait-for-pg.sh"
 
-NODE_ENV=development NODE_NO_WARNINGS=1 REALM_SECRET_SEED="shhh! it's a secret" ts-node \
+wait_for_postgres
+
+NODE_ENV=development \
+  NODE_NO_WARNINGS=1 \
+  PGPORT=5435 \
+  PGDATABASE=boxel_test_base_root \
+  REALM_SECRET_SEED="shhh! it's a secret" \
+  ts-node \
   --transpileOnly main \
   --port=4203 \
   \

packages/realm-server/scripts/start-base.sh

@@ -1,8 +1,16 @@
 #! /bin/sh
+SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+. "$SCRIPTS_DIR/wait-for-pg.sh"
 
-pnpm run setup:base-assets
+wait_for_postgres
 
-NODE_ENV=development NODE_NO_WARNINGS=1 REALM_SECRET_SEED="shhh! it's a secret" ts-node \
+pnpm run setup:base-assets
+NODE_ENV=development \
+  NODE_NO_WARNINGS=1 \
+  PGPORT=5435 \
+  PGDATABASE=boxel_dev_base \
+  REALM_SECRET_SEED="shhh! it's a secret" \
+  ts-node \
   --transpileOnly main \
   --port=4201 \
   \

packages/realm-server/scripts/start-development.sh

@@ -1,10 +1,16 @@
 #! /bin/sh
+SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+. "$SCRIPTS_DIR/wait-for-pg.sh"
+
+wait_for_postgres
+
 pnpm setup:base-assets
 NODE_ENV=development \
   NODE_NO_WARNINGS=1 \
+  PGPORT=5435 \
+  PGDATABASE=boxel_dev \
   LOG_LEVELS='*=info' \
   REALM_SECRET_SEED="shhh! it's a secret" \
-  PGPORT="5435" \
   ts-node \
   --transpileOnly main \
   --port=4201 \

packages/realm-server/scripts/start-drafts-root.sh

@@ -1,6 +1,14 @@
 #! /bin/sh
+SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+. "$SCRIPTS_DIR/wait-for-pg.sh"
 
-NODE_NO_WARNINGS=1 REALM_SECRET_SEED="shhh! it's a secret" ts-node \
+wait_for_postgres
+
+NODE_NO_WARNINGS=1 \
+  PGPORT=5435 \
+  PGDATABASE=boxel_test_drafts_root \
+  REALM_SECRET_SEED="shhh! it's a secret" \
+  ts-node \
   --transpileOnly main \
   --port=4204 \
   \

packages/realm-server/scripts/start-production.sh

@@ -2,7 +2,9 @@
 pnpm setup:base-in-deployment
 pnpm setup:drafts-in-deployment
 pnpm setup:published-in-deployment
-NODE_NO_WARNINGS=1 LOG_LEVELS='*=info' ts-node \
+NODE_NO_WARNINGS=1 \
+  LOG_LEVELS='*=info' \
+  ts-node \
   --transpileOnly main \
   --port=3000 \
   \

packages/realm-server/scripts/start-staging.sh

@@ -2,7 +2,9 @@
 pnpm setup:base-in-deployment
 pnpm setup:drafts-in-deployment
 pnpm setup:published-in-deployment
-NODE_NO_WARNINGS=1 LOG_LEVELS='*=info' ts-node \
+NODE_NO_WARNINGS=1 \
+  LOG_LEVELS='*=info' \
+  ts-node \
   --transpileOnly main \
   --port=3000 \
   \

packages/realm-server/scripts/start-test-realms.sh

@@ -1,6 +1,18 @@
 #! /bin/sh
+check_postgres_ready() {
+  docker exec boxel-pg pg_isready -U postgres >/dev/null 2>&1
+}
+# remove this check after the feature flag is removed
+if [ -n "$PG_INDEXER" ]; then
+  while ! check_postgres_ready; do
+    printf '.'
+    sleep 1
+  done
+fi
 
 NODE_ENV=test \
+  PGPORT=5435 \
+  PGDATABASE=boxel_test \
   NODE_NO_WARNINGS=1 \
   REALM_SECRET_SEED="shhh! it's a secret" \
   PGPORT="5435" \