setting PGPORT and PGDATABSE specifically for each server and allowing index WIP requests to talk to server while it is being built up

Author: habdelra

.github/workflows/ci.yaml

@@ -374,9 +374,10 @@ jobs:
         env:
           NODE_OPTIONS: --max_old_space_size=4096
         working-directory: packages/host
-      - name: Start matrix
-        run: pnpm start:matrix &
-        working-directory: packages/realm-server
+      # TODO we shouldn't have to start matrix separately
+      # - name: Start matrix
+      #   run: pnpm start:matrix &
+      #   working-directory: packages/realm-server
       - name: Start realm servers
         run: PG_INDEXER=true pnpm start:all &
         working-directory: packages/realm-server

README.md

@@ -115,12 +115,12 @@ Boxel uses a Postgres database. In development, the Postgres database runs withi
 
 When running tests we isolate the database between each test run by actually creating a new database for each test with a random database name (e.g. `test_db_1234567`). The test databases are dropped before the beginning of each test run.
 
-If you wish to drop the development database you can execute:
+If you wish to drop the development databases you can execute:
 ```
-pnpm drop-db
+pnpm drop-all-dbs
 ```
 
-You can then run `pnpm migrate up` or start the realm server to create the database again.
+You can then run `PGDATABASE=boxel_dev pnpm migrate up` (with `PGDATABASE` set accordingly) or start the realm server to create the database again.
 
 #### DB Migrations
 When the realm server starts up it will automatically run DB migrations that live in the `packages/realm-server/migrations` folder. As part of development you may wish to run migrations manually as well as to create a new migration.

packages/matrix/scripts/register-realm-users.sh

@@ -1,14 +1,9 @@
 #! /bin/sh
-set -x
-
-echo "starting register realm users"
 
 until $(curl --output /dev/null --silent --head --fail http://localhost:8008); do
   printf '.'
-  curl_output=$(curl --head --fail http://localhost:8008)
   sleep 5
 done
-echo "matrix server is up"
 MATRIX_USERNAME=base_realm MATRIX_PASSWORD=password ts-node --transpileOnly ./scripts/register-test-user.ts
 MATRIX_USERNAME=drafts_realm MATRIX_PASSWORD=password ts-node --transpileOnly ./scripts/register-test-user.ts
 MATRIX_USERNAME=published_realm MATRIX_PASSWORD=password ts-node --transpileOnly ./scripts/register-test-user.ts

packages/realm-server/Dockerfile

@@ -21,4 +21,5 @@ RUN CI=1 pnpm install -r --offline
 
 EXPOSE 3000
 
+# TODO need to set the PG ENV vars to connect to the DB in the command below
 CMD pnpm --filter "./packages/realm-server" $realm_server_script

packages/realm-server/package.json

@@ -80,13 +80,13 @@
     "setup:drafts-in-deployment": "mkdir -p /persistent/drafts && cp --verbose --update --recursive ../drafts-realm/. /persistent/drafts/",
     "setup:published-in-deployment": "mkdir -p /persistent/published && cp --verbose --update --recursive ../published-realm/. /persistent/published/",
     "setup:base-assets": "ts-node --transpileOnly ./scripts/setup-base.ts",
-    "start": "PGPORT=5435 NODE_NO_WARNINGS=1 ts-node --transpileOnly main",
+    "start": "PGPORT=5435 PGDATABASE=boxel_dev NODE_NO_WARNINGS=1 ts-node --transpileOnly main",
     "start:base": "./scripts/start-base.sh",
     "start:test-realms": "./scripts/start-test-realms.sh",
     "start:base:root": "./scripts/start-base-root.sh",
     "start:drafts:root": "./scripts/start-drafts-root.sh",
-    "start:all": "PGPORT=5435 ./scripts/start-all.sh",
-    "start:without-matrix": "PGPORT=5435 ./scripts/start-without-matrix.sh",
+    "start:all": "./scripts/start-all.sh",
+    "start:without-matrix": "./scripts/start-without-matrix.sh",
     "start:staging": "./scripts/start-staging.sh",
     "start:development": "./scripts/start-development.sh",
     "start:production": "./scripts/start-production.sh",
@@ -98,7 +98,8 @@
     "lint:glint": "glint",
     "migrate": "PGDATABASE=boxel ./scripts/ensure-db-exists.sh && PGPORT=5435 PGDATABASE=boxel PGUSER=postgres node-pg-migrate",
     "make-schema": "./scripts/schema-dump.sh",
-    "drop-db": "docker exec boxel-pg dropdb -U postgres -w boxel"
+    "drop-db": "docker exec boxel-pg dropdb -U postgres -w",
+    "drop-all-dbs": "./scripts/drop-all-dbs.sh"
   },
   "volta": {
     "extends": "../../package.json"

packages/realm-server/scripts/drop-all-dbs.sh

@@ -0,0 +1,8 @@
+#! /bin/sh
+
+pnpm run drop-db boxel_dev
+pnpm run drop-db boxel_test
+pnpm run drop-db boxel_dev_base
+pnpm run drop-db boxel_test_base_root
+pnpm run drop-db boxel_test_drafts_root
+pnpm run drop-db boxel_test_drafts_root

packages/realm-server/scripts/start-base-root.sh

@@ -1,6 +1,11 @@
 #! /bin/sh
 
-NODE_ENV=development NODE_NO_WARNINGS=1 REALM_SECRET_SEED="shhh! it's a secret" ts-node \
+NODE_ENV=development \
+  NODE_NO_WARNINGS=1 \
+  PGPORT=5435 \
+  PGDATABASE=boxel_test_base_root \
+  REALM_SECRET_SEED="shhh! it's a secret" \
+  ts-node \
   --transpileOnly main \
   --port=4203 \
   \

packages/realm-server/scripts/start-base.sh

@@ -2,7 +2,12 @@
 
 pnpm run setup:base-assets
 
-NODE_ENV=development NODE_NO_WARNINGS=1 REALM_SECRET_SEED="shhh! it's a secret" ts-node \
+NODE_ENV=development \
+  NODE_NO_WARNINGS=1 \
+  PGPORT=5435 \
+  PGDATABASE=boxel_dev_base \
+  REALM_SECRET_SEED="shhh! it's a secret" \
+  ts-node \
   --transpileOnly main \
   --port=4201 \
   \

packages/realm-server/scripts/start-development.sh

@@ -2,9 +2,10 @@
 pnpm setup:base-assets
 NODE_ENV=development \
   NODE_NO_WARNINGS=1 \
+  PGPORT=5435 \
+  PGDATABASE=boxel_dev \
   LOG_LEVELS='*=info' \
   REALM_SECRET_SEED="shhh! it's a secret" \
-  PGPORT="5435" \
   ts-node \
   --transpileOnly main \
   --port=4201 \

packages/realm-server/scripts/start-drafts-root.sh

@@ -1,6 +1,10 @@
 #! /bin/sh
 
-NODE_NO_WARNINGS=1 REALM_SECRET_SEED="shhh! it's a secret" ts-node \
+NODE_NO_WARNINGS=1 \
+  PGPORT=5435 \
+  PGDATABASE=boxel_test_drafts_root \
+  REALM_SECRET_SEED="shhh! it's a secret" \
+  ts-node \
   --transpileOnly main \
   --port=4204 \
   \

packages/realm-server/scripts/start-production.sh

@@ -2,7 +2,9 @@
 pnpm setup:base-in-deployment
 pnpm setup:drafts-in-deployment
 pnpm setup:published-in-deployment
-NODE_NO_WARNINGS=1 LOG_LEVELS='*=info' ts-node \
+NODE_NO_WARNINGS=1 \
+  LOG_LEVELS='*=info' \
+  ts-node \
   --transpileOnly main \
   --port=3000 \
   \

packages/realm-server/scripts/start-staging.sh

@@ -2,7 +2,9 @@
 pnpm setup:base-in-deployment
 pnpm setup:drafts-in-deployment
 pnpm setup:published-in-deployment
-NODE_NO_WARNINGS=1 LOG_LEVELS='*=info' ts-node \
+NODE_NO_WARNINGS=1 \
+  LOG_LEVELS='*=info' \
+  ts-node \
   --transpileOnly main \
   --port=3000 \
   \

packages/realm-server/scripts/start-test-realms.sh

@@ -1,6 +1,8 @@
 #! /bin/sh
 
 NODE_ENV=test \
+  PGPORT=5435 \
+  PGDATABASE=boxel_test \
   NODE_NO_WARNINGS=1 \
   REALM_SECRET_SEED="shhh! it's a secret" \
   PGPORT="5435" \

packages/runtime-common/realm.ts

@@ -837,6 +837,8 @@ export class Realm {
     if (redirectResponse) {
       return redirectResponse;
     }
+    // allow any WIP index requests to query the index while it's building up
+    isLocal = isLocal || Boolean(request.headers.get('X-Boxel-Use-WIP-Index'));
 
     try {
       // local requests are allowed to query the realm as the index is being built up

packages/runtime-common/worker.ts

@@ -87,7 +87,6 @@ export class Worker {
     this.runnerOptsMgr = runnerOptsManager;
     this.#runner = indexRunner;
     this.#loader = Loader.cloneLoader(loader);
-    this.#realmAdapter.setLoader?.(this.#loader);
   }
 
   async run() {