Improve use of msgtype on CommandResult matrix events

- now used to identify results with a result card vs those without

Author: lukemelia

packages/ai-bot/helpers.ts

@@ -16,7 +16,10 @@ import { MatrixEvent, type IRoomEvent } from 'matrix-js-sdk';
 import { ChatCompletionMessageToolCall } from 'openai/resources/chat/completions';
 import * as Sentry from '@sentry/node';
 import { logger } from '@cardstack/runtime-common';
-import { APP_BOXEL_COMMAND_RESULT_EVENT_TYPE } from '../runtime-common/matrix-constants';
+import {
+  APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
+} from '../runtime-common/matrix-constants';
 import {
   APP_BOXEL_CARDFRAGMENT_MSGTYPE,
   APP_BOXEL_MESSAGE_MSGTYPE,
@@ -139,14 +142,15 @@ export function constructHistory(
       }
     }
     let event = { ...rawEvent } as DiscreteMatrixEvent;
-    if (event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE) {
+    if (
+      event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
+      event.content.msgtype == APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
+    ) {
       let { cardEventId } = event.content.data;
-      if (cardEventId) {
-        event.content.data.card = serializedCardFromFragments(
-          cardEventId,
-          cardFragments,
-        );
-      }
+      event.content.data.card = serializedCardFromFragments(
+        cardEventId,
+        cardFragments,
+      );
     }
     if (event.type !== 'm.room.message') {
       continue;
@@ -401,7 +405,10 @@ function toPromptMessageWithToolResult(
   let content = 'pending';
   if (commandResult) {
     let status = commandResult.content['m.relates_to']?.key;
-    if (commandResult.content.data.card) {
+    if (
+      commandResult.content.msgtype ===
+      APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
+    ) {
       content = `Command ${status}, with result card: ${JSON.stringify(
         commandResult.content.data.card,
       )}.\n`;

packages/ai-bot/tests/chat-titling-test.ts

@@ -4,6 +4,7 @@ import type { MatrixEvent as DiscreteMatrixEvent } from 'https://cardstack.com/b
 import {
   APP_BOXEL_COMMAND_MSGTYPE,
   APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE,
 } from '@cardstack/runtime-common/matrix-constants';
 import { IEvent, IRoomEvent, MatrixEvent } from 'matrix-js-sdk';
 
@@ -415,8 +416,7 @@ module('shouldSetRoomTitle', () => {
           key: 'applied',
           rel_type: 'm.annotation',
         },
-        data: {},
-        msgtype: APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
+        msgtype: APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE,
       },
     };
     const eventLog: IRoomEvent[] = [

packages/ai-bot/tests/prompt-construction-test.ts

@@ -13,9 +13,9 @@ import {
 } from '../helpers';
 import {
   APP_BOXEL_MESSAGE_MSGTYPE,
-  APP_BOXEL_COMMAND_RESULT_MSGTYPE,
   APP_BOXEL_COMMAND_MSGTYPE,
   APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
 } from '@cardstack/runtime-common/matrix-constants';
 
 import type {
@@ -1510,7 +1510,7 @@ test('Return host result of tool call back to open ai', () => {
           rel_type: 'm.annotation',
           key: 'applied',
         },
-        msgtype: APP_BOXEL_COMMAND_RESULT_MSGTYPE,
+        msgtype: APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
         data: {
           card: JSON.stringify({
             data: {

packages/base/matrix-event.gts

@@ -10,7 +10,8 @@ import {
   APP_BOXEL_CARDFRAGMENT_MSGTYPE,
   APP_BOXEL_COMMAND_MSGTYPE,
   APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
-  APP_BOXEL_COMMAND_RESULT_MSGTYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
   APP_BOXEL_MESSAGE_MSGTYPE,
   APP_BOXEL_ROOM_SKILLS_EVENT_TYPE,
 } from '@cardstack/runtime-common/matrix-constants';
@@ -228,7 +229,7 @@ export interface SkillsConfigEvent extends RoomStateEvent {
 
 export interface CommandResultEvent extends BaseMatrixEvent {
   type: typeof APP_BOXEL_COMMAND_RESULT_EVENT_TYPE;
-  content: CommandResultContent;
+  content: CommandResultWithOutputContent | CommandResultWithNoOutputContent;
   unsigned: {
     age: number;
     transaction_id: string;
@@ -237,23 +238,27 @@ export interface CommandResultEvent extends BaseMatrixEvent {
   };
 }
 
-export interface CommandResultContent {
-  'm.relates_to'?: {
+export interface CommandResultWithOutputContent {
+  'm.relates_to': {
     rel_type: 'm.annotation';
     key: string;
     event_id: string;
-    'm.in_reply_to'?: {
-      event_id: string;
-    };
   };
   data: {
-    // we use this field over the wire since the matrix message protocol
-    // limits us to 65KB per message
-    cardEventId?: string;
+    cardEventId: string;
     // we materialize this field on the server
     card?: LooseSingleCardDocument;
   };
-  msgtype: typeof APP_BOXEL_COMMAND_RESULT_MSGTYPE;
+  msgtype: typeof APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE;
+}
+
+export interface CommandResultWithNoOutputContent {
+  'm.relates_to': {
+    rel_type: 'm.annotation';
+    key: string;
+    event_id: string;
+  };
+  msgtype: typeof APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE;
 }
 
 export type MatrixEvent =

packages/host/app/lib/matrix-classes/message-builder.ts

@@ -9,7 +9,7 @@ import { LooseSingleCardDocument } from '@cardstack/runtime-common';
 import {
   APP_BOXEL_COMMAND_MSGTYPE,
   APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
-  APP_BOXEL_COMMAND_RESULT_MSGTYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
   APP_BOXEL_MESSAGE_MSGTYPE,
 } from '@cardstack/runtime-common/matrix-constants';
 
@@ -22,7 +22,6 @@ import type {
   CardMessageEvent,
   CommandEvent,
   CommandResultEvent,
-  CommandResultContent,
   MatrixEvent as DiscreteMatrixEvent,
   MessageEvent,
 } from 'https://cardstack.com/base/matrix-event';
@@ -138,26 +137,23 @@ export default class MessageBuilder {
   private async buildMessageCommand(message: Message) {
     let event = this.event as CommandEvent;
     let command = event.content.data.toolCall;
-    let annotation = this.builderContext.events.find((e: any) => {
+    let commandResultEvent = this.builderContext.events.find((e: any) => {
       let r = e.content['m.relates_to'];
       return (
         e.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
-        e.content.msgtype === APP_BOXEL_COMMAND_RESULT_MSGTYPE &&
-        r?.rel_type === 'm.annotation' &&
-        (r?.event_id === event.content.data.eventId ||
-          r?.event_id === event.event_id ||
-          r?.event_id === this.builderContext.effectiveEventId)
+        r.rel_type === 'm.annotation' &&
+        (r.event_id === event.content.data.eventId ||
+          r.event_id === event.event_id ||
+          r.event_id === this.builderContext.effectiveEventId)
       );
     }) as CommandResultEvent | undefined;
-    let status: CommandStatus = 'ready';
-    let commandResultContent = annotation?.content as
-      | CommandResultContent
-      | undefined;
-    if (commandResultContent?.['m.relates_to']?.key === 'applied') {
-      status = 'applied';
-    }
-    let commandResultCardEventId: string | undefined =
-      commandResultContent?.data.cardEventId ?? undefined;
+    let status = (commandResultEvent?.content?.['m.relates_to']?.key ||
+      'ready') as CommandStatus;
+    let commandResultCardEventId =
+      commandResultEvent?.content?.msgtype ===
+      APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
+        ? commandResultEvent.content.data.cardEventId
+        : undefined;
     let messageCommand = new MessageCommand(
       message,
       command.id,

packages/host/app/services/matrix-service.ts

@@ -41,7 +41,8 @@ import {
   APP_BOXEL_CARDFRAGMENT_MSGTYPE,
   APP_BOXEL_COMMAND_MSGTYPE,
   APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
-  APP_BOXEL_COMMAND_RESULT_MSGTYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
   APP_BOXEL_MESSAGE_MSGTYPE,
   APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE,
   APP_BOXEL_REALMS_EVENT_TYPE,
@@ -64,8 +65,9 @@ import type * as CardAPI from 'https://cardstack.com/base/card-api';
 import type {
   CardMessageContent,
   CardFragmentContent,
-  CommandResultContent,
   MatrixEvent as DiscreteMatrixEvent,
+  CommandResultWithNoOutputContent,
+  CommandResultWithOutputContent,
 } from 'https://cardstack.com/base/matrix-event';
 
 import { SkillCard } from 'https://cardstack.com/base/skill-card';
@@ -501,7 +503,11 @@ export default class MatrixService extends Service {
   async sendEvent(
     roomId: string,
     eventType: string,
-    content: CardMessageContent | CardFragmentContent | CommandResultContent,
+    content:
+      | CardMessageContent
+      | CardFragmentContent
+      | CommandResultWithNoOutputContent
+      | CommandResultWithOutputContent,
   ) {
     let roomData = await this.ensureRoomData(roomId);
     return roomData.mutex.dispatch(async () => {
@@ -526,17 +532,31 @@ export default class MatrixService extends Service {
     if (resultCard) {
       [resultCardEventId] = await this.addCardsToRoom([resultCard], roomId);
     }
-    let content: CommandResultContent = {
-      msgtype: APP_BOXEL_COMMAND_RESULT_MSGTYPE,
-      'm.relates_to': {
-        event_id: invokedToolFromEventId,
-        key: 'applied',
-        rel_type: 'm.annotation',
-      },
-      data: {
-        cardEventId: resultCardEventId ?? undefined,
-      },
-    };
+    let content:
+      | CommandResultWithNoOutputContent
+      | CommandResultWithOutputContent;
+    if (resultCardEventId === undefined) {
+      content = {
+        msgtype: APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE,
+        'm.relates_to': {
+          event_id: invokedToolFromEventId,
+          key: 'applied',
+          rel_type: 'm.annotation',
+        },
+      };
+    } else {
+      content = {
+        msgtype: APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
+        'm.relates_to': {
+          event_id: invokedToolFromEventId,
+          key: 'applied',
+          rel_type: 'm.annotation',
+        },
+        data: {
+          cardEventId: resultCardEventId,
+        },
+      };
+    }
     try {
       return await this.sendEvent(
         roomId,
@@ -1279,16 +1299,14 @@ export default class MatrixService extends Service {
     } else if (
       roomData &&
       event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
-      event.content?.msgtype === APP_BOXEL_COMMAND_RESULT_MSGTYPE
+      event.content?.msgtype === APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
     ) {
       let data = (
         typeof event.content.data === 'string'
           ? JSON.parse(event.content.data)
           : event.content.data
-      ) as CommandResultContent['data'];
-      if (data.cardEventId) {
-        this.ensureCardFragmentsLoaded(data.cardEventId, roomData);
-      }
+      ) as CommandResultWithOutputContent['data'];
+      this.ensureCardFragmentsLoaded(data.cardEventId, roomData);
     } else if (
       event.type === 'm.room.message' &&
       event.content?.msgtype === APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE

packages/host/tests/integration/components/ai-assistant-panel-test.gts

@@ -20,7 +20,6 @@ import {
   APP_BOXEL_CARDFRAGMENT_MSGTYPE,
   APP_BOXEL_COMMAND_MSGTYPE,
   APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
-  APP_BOXEL_COMMAND_RESULT_MSGTYPE,
   APP_BOXEL_MESSAGE_MSGTYPE,
 } from '@cardstack/runtime-common/matrix-constants';
 
@@ -1982,11 +1981,8 @@ module('Integration | ai-assistant-panel', function (hooks) {
         event_id: '__EVENT_ID__',
       },
     });
-    let commandResultEvents = getRoomEvents(roomId).filter(
-      (event) =>
-        event.type === 'm.room.message' &&
-        typeof event.content === 'object' &&
-        event.content.msgtype === APP_BOXEL_COMMAND_RESULT_MSGTYPE,
+    let commandResultEvents = await getRoomEvents(roomId).filter(
+      (event) => event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
     );
     assert.equal(
       commandResultEvents.length,
@@ -2005,10 +2001,7 @@ module('Integration | ai-assistant-panel', function (hooks) {
       .exists();
 
     commandResultEvents = await getRoomEvents(roomId).filter(
-      (event) =>
-        event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
-        typeof event.content === 'object' &&
-        event.content.msgtype === APP_BOXEL_COMMAND_RESULT_MSGTYPE,
+      (event) => event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
     );
     assert.equal(
       commandResultEvents.length,

packages/matrix/helpers/matrix-constants.ts

@@ -3,7 +3,10 @@ export const APP_BOXEL_MESSAGE_MSGTYPE = 'app.boxel.message';
 export const APP_BOXEL_COMMAND_MSGTYPE = 'app.boxel.command';
 export const APP_BOXEL_CARD_FORMAT = 'app.boxel.card';
 export const APP_BOXEL_COMMAND_RESULT_EVENT_TYPE = 'app.boxel.commandResult';
-export const APP_BOXEL_COMMAND_RESULT_MSGTYPE = 'app.boxel.commandResult';
+export const APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE =
+  'app.boxel.commandResultWithOutput';
+export const APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE =
+  'app.boxel.commandResultWithNoOutput';
 export const APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE =
   'app.boxel.realm-server-event';
 export const APP_BOXEL_REALMS_EVENT_TYPE = 'app.boxel.realms';

packages/runtime-common/matrix-constants.ts

@@ -3,7 +3,10 @@ export const APP_BOXEL_MESSAGE_MSGTYPE = 'app.boxel.message';
 export const APP_BOXEL_COMMAND_MSGTYPE = 'app.boxel.command';
 export const APP_BOXEL_CARD_FORMAT = 'app.boxel.card';
 export const APP_BOXEL_COMMAND_RESULT_EVENT_TYPE = 'app.boxel.commandResult';
-export const APP_BOXEL_COMMAND_RESULT_MSGTYPE = 'app.boxel.commandResult';
+export const APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE =
+  'app.boxel.commandResultWithOutput';
+export const APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE =
+  'app.boxel.commandResultWithNoOutput';
 export const APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE =
   'app.boxel.realm-server-event';
 export const APP_BOXEL_ROOM_SKILLS_EVENT_TYPE = 'app.boxel.room.skills';