Skip to content

Commit 24de03d

Browse files
habdelrahabdelra
authored
Merge pull request #1108 from cardstack/cs-6626-implement-sqlite-indexer
SQLite indexer client
2 parents 81cc00e + a95c508 commit 24de03d

File tree

15 files changed

+1635
-96
lines changed

15 files changed

+1635
-96
lines changed

‎packages/host/app/config/environment.d.ts

+1
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,5 @@ declare const config: {
2727
serverEchoDebounceMs: number;
2828
loginMessageTimeoutMs: number;
2929
minSaveTaskDurationMs: number;
30+
sqlSchema: string;
3031
};

‎packages/host/app/lib/SQLiteAdapter.ts

+140
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
import {
2+
sqlite3Worker1Promiser,
3+
type SQLiteWorker,
4+
} from '@sqlite.org/sqlite-wasm';
5+
6+
import {
7+
type DBAdapter,
8+
type PgPrimitive,
9+
type ExecuteOptions,
10+
Deferred,
11+
} from '@cardstack/runtime-common';
12+
13+
export default class SQLiteAdapter implements DBAdapter {
14+
private _sqlite: typeof SQLiteWorker | undefined;
15+
private _dbId: string | undefined;
16+
17+
constructor(private schemaSQL?: string) {}
18+
19+
async startClient() {
20+
let ready = new Deferred<typeof SQLiteWorker>();
21+
const promisedWorker = sqlite3Worker1Promiser({
22+
onready: () => ready.fulfill(promisedWorker),
23+
});
24+
this._sqlite = await ready.promise;
25+
26+
let response = await this.sqlite('open', {
27+
// It is possible to write to the local
28+
// filesystem via Origin Private Filesystem, but it requires _very_
29+
// restrictive response headers that would cause our host app to break
30+
// "Cross-Origin-Embedder-Policy: require-corp"
31+
// "Cross-Origin-Opener-Policy: same-origin"
32+
// https://webkit.org/blog/12257/the-file-system-access-api-with-origin-private-file-system/
33+
34+
// Otherwise, local storage and session storage are off limits to the
35+
// worker (they are available in the synchronous interface), so only
36+
// ephemeral memory storage is available
37+
filename: ':memory:',
38+
});
39+
const { dbId } = response;
40+
this._dbId = dbId;
41+
42+
if (this.schemaSQL) {
43+
try {
44+
await this.sqlite('exec', {
45+
dbId: this.dbId,
46+
sql: this.schemaSQL,
47+
});
48+
} catch (e: any) {
49+
console.error(
50+
`Error executing SQL: ${e.result.message}\n${this.schemaSQL}`,
51+
e,
52+
);
53+
throw e;
54+
}
55+
}
56+
}
57+
58+
async execute(sql: string, opts?: ExecuteOptions) {
59+
return await this.query(sql, opts);
60+
}
61+
62+
async close() {
63+
await this.sqlite('close', { dbId: this.dbId });
64+
}
65+
66+
private get sqlite() {
67+
if (!this._sqlite) {
68+
throw new Error(
69+
`could not get sqlite worker--has createClient() been run?`,
70+
);
71+
}
72+
return this._sqlite;
73+
}
74+
75+
private get dbId() {
76+
if (!this._dbId) {
77+
throw new Error(
78+
`could not obtain db identifier--has createClient() been run?`,
79+
);
80+
}
81+
return this._dbId;
82+
}
83+
84+
private async query(sql: string, opts?: ExecuteOptions) {
85+
let results: Record<string, PgPrimitive>[] = [];
86+
try {
87+
await this.sqlite('exec', {
88+
dbId: this.dbId,
89+
sql,
90+
bind: opts?.bind,
91+
// Nested execs are not possible with this async interface--we can't call
92+
// into the exec in this callback due to the way we communicate to the
93+
// worker thread via postMessage. if we need nesting do it all in the SQL
94+
callback: ({ columnNames, row }) => {
95+
let rowObject: Record<string, any> = {};
96+
// row === undefined indicates that the end of the result set has been reached
97+
if (row) {
98+
for (let [index, col] of columnNames.entries()) {
99+
let coerceAs = opts?.coerceTypes?.[col];
100+
if (coerceAs) {
101+
switch (coerceAs) {
102+
case 'JSON': {
103+
rowObject[col] = JSON.parse(row[index]);
104+
break;
105+
}
106+
case 'BOOLEAN': {
107+
let value = row[index];
108+
rowObject[col] =
109+
// respect DB NULL values
110+
value === null ? value : Boolean(row[index]);
111+
break;
112+
}
113+
default:
114+
assertNever(coerceAs);
115+
}
116+
} else {
117+
rowObject[col] = row[index];
118+
}
119+
}
120+
results.push(rowObject);
121+
}
122+
},
123+
});
124+
} catch (e: any) {
125+
console.error(
126+
`Error executing SQL ${e.result.message}:\n${sql}${
127+
opts?.bind ? ' with bindings: ' + JSON.stringify(opts?.bind) : ''
128+
}`,
129+
e,
130+
);
131+
throw e;
132+
}
133+
134+
return results;
135+
}
136+
}
137+
138+
function assertNever(value: never) {
139+
return new Error(`should never happen ${value}`);
140+
}

‎packages/host/config/environment.js

+6
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
'use strict';
22

3+
const fs = require('fs');
4+
const path = require('path');
5+
6+
let sqlSchema = fs.readFileSync(path.join(__dirname, 'schema.sql'), 'utf8');
7+
38
module.exports = function (environment) {
49
const ENV = {
510
modulePrefix: '@cardstack/host',
@@ -51,6 +56,7 @@ module.exports = function (environment) {
5156
environment === 'test'
5257
? 'http://test-realm/test/'
5358
: process.env.OWN_REALM_URL || 'http://localhost:4200/',
59+
sqlSchema,
5460
};
5561

5662
if (environment === 'development') {

‎packages/host/config/schema.sql

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
CREATE TABLE IF NOT EXISTS indexed_cards (
2+
card_url TEXT NOT NULL,
3+
realm_version INTEGER NOT NULL,
4+
realm_url TEXT NOT NULL,
5+
-- WARNING SQLite doesn't actually have a JSON type. Rather JSON just falls
6+
-- back to TEXT which can be recognized as JSON via SQLite JSON functions as
7+
-- part of queries
8+
pristine_doc JSON,
9+
search_doc JSON,
10+
error_doc JSON,
11+
deps JSON,
12+
types JSON,
13+
embedded_html TEXT,
14+
isolated_html TEXT,
15+
indexed_at INTEGER,
16+
-- WARNING SQLite doesn't have a BOOLEAN type, but it does recognize TRUE and
17+
-- FALSE. These values will be returned as 1 and 0 in SQLite result sets
18+
is_deleted BOOLEAN,
19+
PRIMARY KEY (card_url, realm_version)
20+
);
21+
22+
CREATE TABLE IF NOT EXISTS realm_versions (
23+
realm_url TEXT NOT NULL PRIMARY KEY,
24+
current_version INTEGER NOT NULL
25+
);
26+
27+
CREATE INDEX IF NOT EXISTS realm_version_idx ON indexed_cards (realm_version);
28+
CREATE INDEX IF NOT EXISTS realm_url_idx ON indexed_cards (realm_url);
29+
CREATE INDEX IF NOT EXISTS current_version_idx ON realm_versions (current_version);

‎packages/host/tests/helpers/index.gts

+1
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ import visitOperatorMode from './visit-operator-mode';
5454

5555
export { percySnapshot };
5656
export { visitOperatorMode };
57+
export * from './indexer';
5758

5859
const waiter = buildWaiter('@cardstack/host/test/helpers/index:onFetch-waiter');
5960

‎packages/host/tests/helpers/indexer.ts

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import {
2+
IndexerDBClient,
3+
asExpressions,
4+
addExplicitParens,
5+
separatedByCommas,
6+
type IndexedCardsTable,
7+
type RealmVersionsTable,
8+
} from '@cardstack/runtime-common';
9+
10+
export async function setupIndex(
11+
client: IndexerDBClient,
12+
versionRows: RealmVersionsTable[],
13+
// only assert that the non-null columns need to be present in rows objects
14+
indexRows: (Pick<
15+
IndexedCardsTable,
16+
'card_url' | 'realm_version' | 'realm_url'
17+
> &
18+
Partial<
19+
Omit<IndexedCardsTable, 'card_url' | 'realm_version' | 'realm_url'>
20+
>)[],
21+
) {
22+
let indexedCardsExpressions = indexRows.map((r) =>
23+
asExpressions(r, {
24+
jsonFields: ['deps', 'types', 'pristine_doc', 'error_doc', 'search_doc'],
25+
}),
26+
);
27+
let versionExpressions = versionRows.map((r) => asExpressions(r));
28+
29+
if (indexedCardsExpressions.length > 0) {
30+
await client.query([
31+
`INSERT INTO indexed_cards`,
32+
...addExplicitParens(
33+
separatedByCommas(indexedCardsExpressions[0].nameExpressions),
34+
),
35+
'VALUES',
36+
...separatedByCommas(
37+
indexedCardsExpressions.map((row) =>
38+
addExplicitParens(separatedByCommas(row.valueExpressions)),
39+
),
40+
),
41+
]);
42+
}
43+
44+
if (versionExpressions.length > 0) {
45+
await client.query([
46+
`INSERT INTO realm_versions`,
47+
...addExplicitParens(
48+
separatedByCommas(versionExpressions[0].nameExpressions),
49+
),
50+
'VALUES',
51+
...separatedByCommas(
52+
versionExpressions.map((row) =>
53+
addExplicitParens(separatedByCommas(row.valueExpressions)),
54+
),
55+
),
56+
]);
57+
}
58+
}

0 commit comments

Comments
 (0)