Move SSE to Matrix (#2116)

Author: backspace

.github/workflows/ci.yaml

@@ -282,6 +282,10 @@ jobs:
       - name: create realm users
         run: pnpm register-realm-users
         working-directory: packages/matrix
+      # TODO: remove with CS-8104
+      - name: Install watchexec
+        run: cargo install --locked watchexec-cli
+        if: ${{ matrix.testModule == 'realm-endpoints-test.ts' }}
       - name: realm server test suite
         run: pnpm test:wait-for-servers
         working-directory: packages/realm-server

packages/base/cards-grid.gts

@@ -45,6 +45,8 @@ import Captions from '@cardstack/boxel-icons/captions';
 import CardsIcon from '@cardstack/boxel-icons/cards';
 import { registerDestructor } from '@ember/destroyable';
 
+import type { RealmEventContent } from 'https://cardstack.com/base/matrix-event';
+
 type IconComponent = typeof Captions;
 interface SortOption {
   displayName: string;
@@ -469,9 +471,8 @@ class Isolated extends Component<typeof CardsGrid> {
       ) ?? this.filters[0];
   });
 
-  private refreshFilterList = (ev: MessageEvent) => {
-    let data = JSON.parse(ev.data);
-    if (ev.type === 'index' && data.type === 'incremental') {
+  private refreshFilterList = (ev: RealmEventContent) => {
+    if (ev.eventName === 'index' && ev.indexType === 'incremental') {
       this.loadFilterList.perform();
     }
   };

packages/base/matrix-event.gts

@@ -16,6 +16,8 @@ import {
   APP_BOXEL_MESSAGE_MSGTYPE,
   APP_BOXEL_ROOM_SKILLS_EVENT_TYPE,
   APP_BOXEL_ACTIVE_LLM,
+  APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE,
+  APP_BOXEL_REALM_EVENT_TYPE,
   APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE,
 } from '@cardstack/runtime-common/matrix-constants';
 import { type SerializedFile } from './file-api';
@@ -274,6 +276,75 @@ export interface CommandResultWithNoOutputContent {
   commandRequestId: string;
 }
 
+export interface RealmServerEvent extends BaseMatrixEvent {
+  type: 'm.room.message';
+  content: RealmServerEventContent;
+}
+
+export interface RealmServerEventContent {
+  msgtype: typeof APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE;
+  body: string;
+}
+
+export interface RealmEvent extends BaseMatrixEvent {
+  type: typeof APP_BOXEL_REALM_EVENT_TYPE;
+  content: RealmEventContent;
+}
+
+export type RealmEventContent =
+  | IndexRealmEventContent
+  | UpdateRealmEventContent;
+
+export type IndexRealmEventContent =
+  | IncrementalIndexEventContent
+  | FullIndexEventContent
+  | CopiedIndexEventContent
+  | IncrementalIndexInitiationContent;
+
+export interface IncrementalIndexEventContent {
+  eventName: 'index';
+  indexType: 'incremental';
+  invalidations: string[];
+  clientRequestId?: string | null;
+}
+
+interface FullIndexEventContent {
+  eventName: 'index';
+  indexType: 'full';
+}
+
+interface CopiedIndexEventContent {
+  eventName: 'index';
+  indexType: 'copy';
+  sourceRealmURL: string;
+}
+
+interface IncrementalIndexInitiationContent {
+  eventName: 'index';
+  indexType: 'incremental-index-initiation';
+  updatedFile: string;
+}
+
+export type UpdateRealmEventContent =
+  | FileAddedEventContent
+  | FileUpdatedEventContent
+  | FileRemovedEventContent;
+
+interface FileAddedEventContent {
+  eventName: 'update';
+  added: string;
+}
+
+interface FileUpdatedEventContent {
+  eventName: 'update';
+  updated: string;
+}
+
+interface FileRemovedEventContent {
+  eventName: 'update';
+  removed: string;
+}
+
 export type MatrixEvent =
   | RoomCreateEvent
   | RoomJoinRules
@@ -282,6 +353,8 @@ export type MatrixEvent =
   | CommandResultEvent
   | CommandDefinitionsEvent
   | CardMessageEvent
+  | RealmServerEvent
+  | RealmEvent
   | RoomNameEvent
   | RoomTopicEvent
   | InviteEvent

packages/host/app/components/prerendered-card-search.gts

@@ -20,6 +20,8 @@ import {
 
 import { type Format } from 'https://cardstack.com/base/card-api';
 
+import type { RealmEventContent } from 'https://cardstack.com/base/matrix-event';
+
 import SubscribeToRealms from '../helpers/subscribe-to-realms';
 import { type HTMLComponent, htmlComponent } from '../lib/html-component';
 
@@ -252,8 +254,8 @@ export default class PrerenderedCardSearch extends Component<Signature> {
     }
   }
 
-  private markRealmNeedsRefreshing = (ev: MessageEvent, realm: string) => {
-    if (ev.type === 'index') {
+  private markRealmNeedsRefreshing = (ev: RealmEventContent, realm: string) => {
+    if (ev.eventName === 'index') {
       this.realmsNeedingRefresh.add(realm);
     }
   };

packages/host/app/helpers/subscribe-to-realms.ts

@@ -4,11 +4,13 @@ import { isEqual } from 'lodash';
 
 import { subscribeToRealm } from '@cardstack/runtime-common';
 
+import type { RealmEventContent } from 'https://cardstack.com/base/matrix-event';
+
 interface Signature {
   Args: {
     Positional: [
       realms: string[],
-      callback: (ev: MessageEvent, realmURL: string) => void,
+      callback: (ev: RealmEventContent, realmURL: string) => void,
     ];
   };
   Return: void;

packages/host/app/resources/directory.ts

@@ -12,6 +12,8 @@ import {
   type Relationship,
 } from '@cardstack/runtime-common';
 
+import type { RealmEventContent } from 'https://cardstack.com/base/matrix-event';
+
 import type LoaderService from '../services/loader-service';
 import type MessageService from '../services/message-service';
 import type NetworkService from '../services/network';
@@ -65,16 +67,20 @@ export class DirectoryResource extends Resource<Args> {
         url: realmURL.href,
         unsubscribe: this.messageService.subscribe(
           realmURL.href,
-          ({ type, data: dataStr }) => {
+          (event: RealmEventContent) => {
             if (!this.directoryURL) {
               return;
             }
-            let eventData = JSON.parse(dataStr);
-            if (type !== 'index' || !eventData.updatedFile) {
+
+            if (event.eventName !== 'index') {
+              return;
+            }
+
+            if (!('updatedFile' in event)) {
               return;
             }
 
-            let { updatedFile } = eventData as { updatedFile: string };
+            let { updatedFile } = event as { updatedFile: string };
             let segments = updatedFile.split('/');
             segments.pop();
             let updatedDir = segments.join('/').replace(/([^/])$/, '$1/'); // directories always end in '/'

packages/host/app/resources/file.ts

@@ -15,6 +15,8 @@ import type CardService from '@cardstack/host/services/card-service';
 import type OperatorModeStateService from '@cardstack/host/services/operator-mode-state-service';
 import type RecentFilesService from '@cardstack/host/services/recent-files-service';
 
+import type { RealmEventContent } from 'https://cardstack.com/base/matrix-event';
+
 import type LoaderService from '../services/loader-service';
 import type MessageService from '../services/message-service';
 import type NetworkService from '../services/network';
@@ -91,7 +93,7 @@ class _FileResource extends Resource<Args> {
 
   private setSubscription(
     realmURL: string,
-    callback: (ev: { type: string; data: string }) => void,
+    callback: (ev: RealmEventContent) => void,
   ) {
     if (this.subscription && this.subscription.url !== realmURL) {
       this.subscription.unsubscribe();
@@ -197,19 +199,18 @@ class _FileResource extends Resource<Args> {
       },
     });
 
-    this.setSubscription(realmURL, (event: { type: string; data: string }) => {
-      let eventData = JSON.parse(event.data);
+    this.setSubscription(realmURL, (event: RealmEventContent) => {
       if (
-        event.type !== 'index' ||
+        event.eventName !== 'index' ||
         // we wait specifically for the index complete event ("incremental") so
         // that the subsequent index read retrieves the latest contents of the file
-        eventData.type !== 'incremental' ||
-        !Array.isArray(eventData.invalidations)
+        event.indexType !== 'incremental' ||
+        !Array.isArray(event.invalidations)
       ) {
         return;
       }
 
-      let { invalidations } = eventData as { invalidations: string[] };
+      let { invalidations } = event as { invalidations: string[] };
       let normalizedURL = this.url.endsWith('.json')
         ? this.url.replace(/\.json$/, '')
         : this.url;

packages/host/app/resources/room.ts

@@ -16,6 +16,7 @@ import {
   APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE,
   APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
   APP_BOXEL_COMMAND_RESULT_REL_TYPE,
+  APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE,
   DEFAULT_LLM,
 } from '@cardstack/runtime-common/matrix-constants';
 
@@ -31,6 +32,7 @@ import type {
   MessageEvent,
   CommandDefinitionsEvent,
   CommandResultEvent,
+  RealmServerEvent,
 } from 'https://cardstack.com/base/matrix-event';
 
 import type { SkillCard } from 'https://cardstack.com/base/skill-card';
@@ -115,7 +117,10 @@ export class RoomResource extends Resource<Args> {
           case 'm.room.message':
             if (this.isCardFragmentEvent(event)) {
               await this.loadCardFragment(event);
-            } else if (this.isCommandDefinitionsEvent(event)) {
+            } else if (
+              this.isCommandDefinitionsEvent(event) ||
+              this.isRealmServerEvent(event)
+            ) {
               break;
             } else {
               await this.loadRoomMessage({
@@ -319,13 +324,31 @@ export class RoomResource extends Resource<Args> {
   }
 
   private isCommandDefinitionsEvent(
-    event: MessageEvent | CardMessageEvent | CommandDefinitionsEvent,
+    event:
+      | MessageEvent
+      | CardMessageEvent
+      | CommandDefinitionsEvent
+      | RealmServerEvent,
   ): event is CommandDefinitionsEvent {
     return event.content.msgtype === APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE;
   }
 
+  private isRealmServerEvent(
+    event:
+      | MessageEvent
+      | CardMessageEvent
+      | CommandDefinitionsEvent
+      | RealmServerEvent,
+  ): event is RealmServerEvent {
+    return event.content.msgtype === APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE;
+  }
+
   private isCardFragmentEvent(
-    event: MessageEvent | CardMessageEvent | CommandDefinitionsEvent,
+    event:
+      | MessageEvent
+      | CardMessageEvent
+      | CommandDefinitionsEvent
+      | RealmServerEvent,
   ): event is CardMessageEvent & {
     content: { msgtype: typeof APP_BOXEL_CARDFRAGMENT_MSGTYPE };
   } {

packages/host/app/resources/search.ts

@@ -21,6 +21,8 @@ import type { Query } from '@cardstack/runtime-common/query';
 
 import type { CardDef } from 'https://cardstack.com/base/card-api';
 
+import type { RealmEventContent } from 'https://cardstack.com/base/matrix-event';
+
 import { asURL } from '../services/store';
 
 import { type CardResource, getCard } from './card-resource';
@@ -64,23 +66,22 @@ export class Search extends Resource<Args> {
     if (isLive) {
       this.subscriptions = this.realmsToSearch.map((realm) => ({
         url: `${realm}_message`,
-        unsubscribe: subscribeToRealm(
-          realm,
-          ({ type, data }: { type: string; data: string }) => {
-            if (query === undefined) {
-              return;
-            }
-            let eventData = JSON.parse(data);
-            // we are only interested in incremental index events
-            if (type !== 'index' || eventData.type !== 'incremental') {
-              return;
-            }
-            this.search.perform(query);
-            if (doWhileRefreshing) {
-              this.doWhileRefreshing.perform(doWhileRefreshing);
-            }
-          },
-        ),
+        unsubscribe: subscribeToRealm(realm, (event: RealmEventContent) => {
+          if (query === undefined) {
+            return;
+          }
+          // we are only interested in incremental index events
+          if (
+            event.eventName !== 'index' ||
+            event.indexType !== 'incremental'
+          ) {
+            return;
+          }
+          this.search.perform(query);
+          if (doWhileRefreshing) {
+            this.doWhileRefreshing.perform(doWhileRefreshing);
+          }
+        }),
       }));
 
       registerDestructor(this, () => {