Skip to content

Commit a5d6fdf

Browse files
authored
Merge pull request #1157 from cardstack/cs-6463-document-and-setup-postgres-for-dev-environments
Introduce Postgres DB Adapter
2 parents 0fc8f68 + b4063ab commit a5d6fdf

20 files changed

+498
-8
lines changed

README.md

+1
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all`
8282
| :4204 | `root (/)` drafts realm || 🚫 |
8383
| :4205 | qunit server mounting realms in iframes for testing || 🚫 |
8484
| :5001 | Mail user interface for viewing emails sent to local SMTP || 🚫 |
85+
| :5435 | Postgres DB || 🚫 |
8586
| :8008 | Matrix synapse server || 🚫 |
8687

8788
#### Using `start:development`
File renamed without changes.

packages/host/tests/unit/indexer-test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import {
99
} from '@cardstack/runtime-common';
1010

1111
import ENV from '@cardstack/host/config/environment';
12-
import SQLiteAdapter from '@cardstack/host/lib/SQLiteAdapter';
12+
import SQLiteAdapter from '@cardstack/host/lib/sqlite-adapter';
1313

1414
import { testRealmURL, setupIndex } from '../helpers';
1515

packages/host/tests/unit/query-test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ import {
1212
} from '@cardstack/runtime-common';
1313

1414
import ENV from '@cardstack/host/config/environment';
15-
import SQLiteAdapter from '@cardstack/host/lib/SQLiteAdapter';
1615
import { shimExternals } from '@cardstack/host/lib/externals';
16+
import SQLiteAdapter from '@cardstack/host/lib/sqlite-adapter';
1717

1818
import { CardDef } from 'https://cardstack.com/base/card-api';
1919

packages/host/tests/unit/sqlite-test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { module, test } from 'qunit';
22

3-
import SQLiteAdapter from '@cardstack/host/lib/SQLiteAdapter';
3+
import SQLiteAdapter from '@cardstack/host/lib/sqlite-adapter';
44

55
module('Unit | sqlite | SQLiteAdapter', function () {
66
test('run a sqlite db using the SQLiteAdapter', async function (assert) {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
'use strict';
2+
3+
module.exports = {
4+
root: true,
5+
parser: '@typescript-eslint/parser',
6+
parserOptions: {
7+
sourceType: 'script',
8+
},
9+
env: {
10+
browser: false,
11+
node: true,
12+
},
13+
extends: ['plugin:n/recommended'],
14+
rules: {
15+
camelcase: 'off',
16+
},
17+
};
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
exports.up = (pgm) => {
2+
pgm.createTable('indexed_cards', {
3+
card_url: { type: 'varchar', notNull: true },
4+
realm_version: { type: 'varchar', notNull: true },
5+
realm_url: { type: 'varchar', notNull: true },
6+
pristine_doc: 'jsonb',
7+
search_doc: 'jsonb',
8+
error_doc: 'jsonb',
9+
deps: 'jsonb',
10+
types: 'jsonb',
11+
embedded_html: 'varchar',
12+
isolated_html: 'varchar',
13+
indexed_at: 'integer',
14+
is_deleted: 'boolean',
15+
});
16+
pgm.sql('ALTER TABLE indexed_cards SET UNLOGGED');
17+
pgm.addConstraint('indexed_cards', 'indexed_cards_pkey', {
18+
primaryKey: ['card_url', 'realm_version'],
19+
});
20+
pgm.createIndex('indexed_cards', ['realm_version']);
21+
pgm.createIndex('indexed_cards', ['realm_url']);
22+
23+
pgm.createTable('realm_versions', {
24+
realm_url: { type: 'varchar', notNull: true },
25+
current_version: { type: 'integer', notNull: true },
26+
});
27+
28+
pgm.sql('ALTER TABLE realm_versions SET UNLOGGED');
29+
pgm.addConstraint('realm_versions', 'realm_versions_pkey', {
30+
primaryKey: ['realm_url'],
31+
});
32+
pgm.createIndex('realm_versions', ['current_version']);
33+
34+
pgm.createType('job_statuses', ['unfulfilled', 'resolved', 'rejected']);
35+
pgm.createTable('jobs', {
36+
id: 'id', // shorthand for primary key that is an auto incremented id
37+
category: {
38+
type: 'varchar',
39+
notNull: true,
40+
},
41+
args: 'jsonb',
42+
status: {
43+
type: 'job_statuses',
44+
default: 'unfulfilled',
45+
notNull: true,
46+
},
47+
created_at: {
48+
type: 'timestamp',
49+
notNull: true,
50+
default: pgm.func('current_timestamp'),
51+
},
52+
finished_at: {
53+
type: 'timestamp',
54+
},
55+
queue: {
56+
type: 'varchar',
57+
notNull: true,
58+
},
59+
result: 'jsonb',
60+
});
61+
pgm.sql('ALTER TABLE jobs SET UNLOGGED');
62+
};

packages/realm-server/package.json

+9-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
"@types/lodash": "^4.14.182",
2222
"@types/mime-types": "^2.1.1",
2323
"@types/node": "^18.18.5",
24+
"@types/node-pg-migrate": "^2.3.1",
25+
"@types/pg": "^8.11.5",
2426
"@types/qs": "^6.9.14",
2527
"@types/qunit": "^2.11.3",
2628
"@types/sane": "^2.0.1",
@@ -46,7 +48,9 @@
4648
"lodash": "^4.17.21",
4749
"loglevel": "^1.8.1",
4850
"mime-types": "^2.1.35",
51+
"node-pg-migrate": "^6.2.2",
4952
"npm-run-all": "^4.1.5",
53+
"pg": "^8.11.5",
5054
"prettier": "^2.8.4",
5155
"prettier-plugin-ember-template-tag": "^1.1.0",
5256
"qs": "^6.10.5",
@@ -63,11 +67,13 @@
6367
"yargs": "^17.5.1"
6468
},
6569
"scripts": {
66-
"test": "NODE_NO_WARNINGS=1 qunit --require ts-node/register/transpile-only tests/index.ts",
70+
"test": "./scripts/remove-test-dbs.sh && NODE_NO_WARNINGS=1 PGPORT=5435 qunit --require ts-node/register/transpile-only tests/index.ts",
6771
"test:dom": "cd ../matrix && pnpm register-test-user && cd ../realm-server && start-server-and-test 'pnpm run wait' 'http://127.0.0.1:4205' 'testem ci -f ./testem.js'",
6872
"start:test-container": "http-server ./dom-tests -p 4205 --silent",
6973
"start:matrix": "cd ../matrix && pnpm assert-synapse-running",
7074
"start:smtp": "cd ../matrix && pnpm assert-smtp-running",
75+
"start:pg": "./scripts/start-pg.sh",
76+
"stop:pg": "./scripts/stop-pg.sh",
7177
"test:wait-for-servers": "NODE_NO_WARNINGS=1 start-server-and-test 'pnpm run wait' 'http-get://localhost:4201/base/fields/boolean-field?acceptHeader=application%2Fvnd.card%2Bjson' 'pnpm run wait' 'http-get://localhost:4202/node-test/person-1?acceptHeader=application%2Fvnd.card%2Bjson|http://localhost:8008|http://localhost:5001' 'test'",
7278
"setup:base-in-deployment": "rm -rf /persistent/base && cp -R ../base /persistent",
7379
"setup:drafts-in-deployment": "mkdir -p /persistent/drafts && cp --verbose --update --recursive ../drafts-realm/. /persistent/drafts/",
@@ -88,7 +94,8 @@
8894
"lint:fix": "concurrently \"pnpm:lint:*:fix\" --names \"fix:\"",
8995
"lint:js": "eslint . --cache",
9096
"lint:js:fix": "eslint . --fix",
91-
"lint:glint": "glint"
97+
"lint:glint": "glint",
98+
"migrate": "node-pg-migrate"
9299
},
93100
"volta": {
94101
"extends": "../../package.json"

packages/realm-server/pg-adapter.ts

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
import {
2+
type DBAdapter,
3+
type PgPrimitive,
4+
type ExecuteOptions,
5+
} from '@cardstack/runtime-common';
6+
import migrate from 'node-pg-migrate';
7+
import { join } from 'path';
8+
import { Pool, Client } from 'pg';
9+
10+
import postgresConfig from './pg-config';
11+
12+
function config() {
13+
return postgresConfig({
14+
database: 'boxel',
15+
});
16+
}
17+
18+
export default class PgAdapter implements DBAdapter {
19+
private pool: Pool;
20+
21+
constructor() {
22+
let { user, host, database, password, port } = config();
23+
console.log(`connecting to DB ${user}@${host}:${port}/${database}`);
24+
this.pool = new Pool({
25+
user,
26+
host,
27+
database,
28+
password,
29+
port,
30+
});
31+
}
32+
33+
async startClient() {
34+
await this.migrateDb();
35+
}
36+
37+
async close() {
38+
if (this.pool) {
39+
await this.pool.end();
40+
}
41+
}
42+
43+
async execute(
44+
sql: string,
45+
opts?: ExecuteOptions,
46+
): Promise<Record<string, PgPrimitive>[]> {
47+
let client = await this.pool.connect();
48+
try {
49+
let { rows } = await client.query({ text: sql, values: opts?.bind });
50+
return rows;
51+
} catch (e: any) {
52+
console.error(
53+
`Error executing SQL ${e.result.message}:\n${sql}${
54+
opts?.bind ? ' with bindings: ' + JSON.stringify(opts?.bind) : ''
55+
}`,
56+
e,
57+
);
58+
throw e;
59+
} finally {
60+
client.release();
61+
}
62+
}
63+
64+
private async migrateDb() {
65+
const config = postgresConfig();
66+
let client = new Client(
67+
Object.assign({}, config, { database: 'postgres' }),
68+
);
69+
try {
70+
await client.connect();
71+
let response = await client.query(
72+
`select count(*)=1 as has_database from pg_database where datname=$1`,
73+
[config.database],
74+
);
75+
if (!response.rows[0].has_database) {
76+
await client.query(`create database ${config.database}`);
77+
}
78+
} finally {
79+
client.end();
80+
}
81+
82+
await migrate({
83+
direction: 'up',
84+
migrationsTable: 'migrations',
85+
singleTransaction: true,
86+
checkOrder: false,
87+
databaseUrl: {
88+
user: config.user,
89+
host: config.host,
90+
database: config.database,
91+
password: config.password,
92+
port: config.port,
93+
},
94+
count: Infinity,
95+
dir: join(__dirname, 'migrations'),
96+
ignorePattern: '.*\\.eslintrc\\.js',
97+
log: (...args) => console.log(...args),
98+
});
99+
}
100+
}

packages/realm-server/pg-config.ts

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import { type ClientConfig } from 'pg';
2+
3+
export default function postgresConfig(defaultConfig: ClientConfig = {}) {
4+
return Object.assign({}, defaultConfig, {
5+
host: process.env.PGHOST || 'localhost',
6+
port: process.env.PGPORT || '5432',
7+
user: process.env.PGUSER || 'postgres',
8+
password: process.env.PGPASSWORD || undefined,
9+
database: process.env.PGDATABASE || 'postgres',
10+
});
11+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/sh
2+
3+
databases=$(docker exec boxel-pg psql -U postgres -w -lqt | cut -d \| -f 1 | grep -E 'test_db_' | tr -d ' ')
4+
5+
for db in $databases; do
6+
docker exec boxel-pg dropdb -U postgres -w $db
7+
done

packages/realm-server/scripts/start-all.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#! /bin/sh
22

33
NODE_NO_WARNINGS=1 start-server-and-test \
4-
'run-p start:matrix start:smtp start:development start:base:root' \
4+
'run-p start:pg start:matrix start:smtp start:development start:base:root' \
55
'http-get://localhost:4201/base/fields/boolean-field?acceptHeader=application%2Fvnd.card%2Bjson|http-get://localhost:4203/fields/boolean-field?acceptHeader=application%2Fvnd.card%2Bjson|http-get://localhost:4201/drafts/index?acceptHeader=application%2Fvnd.card%2Bjson|http://localhost:8008|http://localhost:5001' \
66
'run-p start:test-realms start:test-container' \
77
'http-get://localhost:4202/node-test/person-1?acceptHeader=application%2Fvnd.card%2Bjson|http-get://127.0.0.1:4205' \

packages/realm-server/scripts/start-development.sh

+6-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
#! /bin/sh
22
pnpm setup:base-assets
3-
NODE_ENV=development NODE_NO_WARNINGS=1 LOG_LEVELS='*=info' REALM_SECRET_SEED="shhh! it's a secret" ts-node \
3+
NODE_ENV=development \
4+
NODE_NO_WARNINGS=1 \
5+
LOG_LEVELS='*=info' \
6+
REALM_SECRET_SEED="shhh! it's a secret" \
7+
PGPORT="5435" \
8+
ts-node \
49
--transpileOnly main \
510
--port=4201 \
611
\
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#! /bin/sh
2+
if [ -z "$(docker ps -f name=boxel-pg --all --format '{{.Names}}')" ]; then
3+
# running postgres on port 5435 so it doesn't collide with native postgres
4+
# that may be running on your system
5+
docker run --name boxel-pg -e POSTGRES_HOST_AUTH_METHOD=trust -p 5435:5432 -d postgres >/dev/null
6+
else
7+
docker start boxel-pg >/dev/null
8+
fi

packages/realm-server/scripts/start-test-realms.sh

+5-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
#! /bin/sh
22

3-
NODE_ENV=test NODE_NO_WARNINGS=1 REALM_SECRET_SEED="shhh! it's a secret" ts-node \
3+
NODE_ENV=test \
4+
NODE_NO_WARNINGS=1 \
5+
REALM_SECRET_SEED="shhh! it's a secret" \
6+
PGPORT="5435" \
7+
ts-node \
48
--transpileOnly main \
59
--port=4202 \
610
\
+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#! /bin/sh
2+
docker stop boxel-pg >/dev/null

packages/realm-server/tests/helpers/index.ts

+4
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@ let basePath = resolve(join(__dirname, '..', '..', '..', 'base'));
2828
let manager = new RunnerOptionsManager();
2929
let getRunner: IndexRunner | undefined;
3030

31+
export async function prepareTestDB() {
32+
process.env.PGDATABASE = `test_db_${Math.floor(10000000 * Math.random())}`;
33+
}
34+
3135
export async function createRealm(
3236
loader: Loader,
3337
dir: string,

packages/realm-server/tests/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ import './indexing-test';
55
import './module-syntax-test';
66
import './permissions/permission-checker-test';
77
import './auth-client-test';
8+
import './pg-test';

0 commit comments

Comments
 (0)