Fix how command results are added to prompt when there is no body

Author: lukemelia

packages/ai-bot/helpers.ts

@@ -19,19 +19,17 @@ import { MatrixEvent, type IRoomEvent } from 'matrix-js-sdk';
 import { ChatCompletionMessageToolCall } from 'openai/resources/chat/completions';
 import * as Sentry from '@sentry/node';
 import { logger } from '@cardstack/runtime-common';
-import {
-  APP_BOXEL_COMMAND_REQUESTS_KEY,
-  APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE,
-  APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
-  APP_BOXEL_COMMAND_RESULT_REL_TYPE,
-  APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
-} from '../runtime-common/matrix-constants';
 import {
   APP_BOXEL_CARDFRAGMENT_MSGTYPE,
   APP_BOXEL_MESSAGE_MSGTYPE,
   APP_BOXEL_ROOM_SKILLS_EVENT_TYPE,
   DEFAULT_LLM,
   APP_BOXEL_ACTIVE_LLM,
+  APP_BOXEL_COMMAND_REQUESTS_KEY,
+  APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE,
+  APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
+  APP_BOXEL_COMMAND_RESULT_REL_TYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
 } from '@cardstack/runtime-common/matrix-constants';
 
 let log = logger('ai-bot');
@@ -654,7 +652,7 @@ export async function getModifyPrompt(
       continue;
     }
     if (isCommandResultEvent(event)) {
-      continue;
+      continue; // we'll include these with the tool calls
     }
     if (
       'isStreamingFinished' in event.content &&
@@ -663,40 +661,39 @@ export async function getModifyPrompt(
       continue;
     }
     let body = event.content.body;
-    if (body) {
-      if (event.sender === aiBotUserId) {
-        let toolCalls = toToolCalls(event);
-        let historicalMessage: OpenAIPromptMessage = {
-          role: 'assistant',
-          content: body,
-        };
-        if (toolCalls.length) {
-          historicalMessage.tool_calls = toolCalls;
-        }
-        historicalMessages.push(historicalMessage);
-        if (toolCalls.length) {
-          toPromptMessageWithToolResults(event, history).forEach((message) =>
-            historicalMessages.push(message),
-          );
-        }
-      } else {
-        if (
-          event.content.msgtype === APP_BOXEL_MESSAGE_MSGTYPE &&
-          event.content.data?.context?.openCardIds
-        ) {
-          body = `User message: ${body}
+    if (event.sender === aiBotUserId) {
+      let toolCalls = toToolCalls(event);
+      let historicalMessage: OpenAIPromptMessage = {
+        role: 'assistant',
+        content: body,
+      };
+      if (toolCalls.length) {
+        historicalMessage.tool_calls = toolCalls;
+      }
+      historicalMessages.push(historicalMessage);
+      if (toolCalls.length) {
+        toPromptMessageWithToolResults(event, history).forEach((message) =>
+          historicalMessages.push(message),
+        );
+      }
+    }
+    if (body && event.sender !== aiBotUserId) {
+      if (
+        event.content.msgtype === APP_BOXEL_MESSAGE_MSGTYPE &&
+        event.content.data?.context?.openCardIds
+      ) {
+        body = `User message: ${body}
           Context: the user has the following cards open: ${JSON.stringify(
             event.content.data.context.openCardIds,
           )}`;
-        } else {
-          body = `User message: ${body}
+      } else {
+        body = `User message: ${body}
           Context: the user has no open cards.`;
-        }
-        historicalMessages.push({
-          role: 'user',
-          content: body,
-        });
       }
+      historicalMessages.push({
+        role: 'user',
+        content: body,
+      });
     }
   }
 
@@ -799,27 +796,3 @@ export function isCommandResultEvent(
       APP_BOXEL_COMMAND_RESULT_REL_TYPE
   );
 }
-
-export function eventRequiresResponse(event: MatrixEvent) {
-  // If it's a message, we should respond unless it's a card fragment
-  if (event.getType() === 'm.room.message') {
-    if (
-      event.getContent().msgtype === APP_BOXEL_CARDFRAGMENT_MSGTYPE ||
-      event.getContent().msgtype === APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE
-    ) {
-      return false;
-    }
-    return true;
-  }
-
-  // If it's a command result with output, we should respond
-  if (
-    event.getType() === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
-    event.getContent().msgtype === APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
-  ) {
-    return true;
-  }
-
-  // If it's a different type, or a command result without output, we should not respond
-  return false;
-}

packages/ai-bot/lib/responder.ts

@@ -12,11 +12,47 @@ import { CommandRequest } from '@cardstack/runtime-common/commands';
 import { thinkingMessage } from '../constants';
 import type OpenAI from 'openai';
 import type { ChatCompletionSnapshot } from 'openai/lib/ChatCompletionStream';
+import {
+  APP_BOXEL_CARDFRAGMENT_MSGTYPE,
+  APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE,
+  APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
+} from '@cardstack/runtime-common/matrix-constants';
 
 let log = logger('ai-bot');
 
 export class Responder {
-  // internally has a debounced function that will send the matrix messages
+  static eventMayTriggerResponse(event: DiscreteMatrixEvent) {
+    // If it's a message, we should respond unless it's a card fragment
+    if (event.getType() === 'm.room.message') {
+      if (
+        event.getContent().msgtype === APP_BOXEL_CARDFRAGMENT_MSGTYPE ||
+        event.getContent().msgtype === APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE
+      ) {
+        return false;
+      }
+      return true;
+    }
+
+    // If it's a command result with output, we might respond
+    if (
+      event.getType() === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
+      event.getContent().msgtype ===
+        APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
+    ) {
+      return true;
+    }
+
+    // If it's a different type, or a command result without output, we should not respond
+    return false;
+  }
+
+  static eventWillDefinitelyTriggerResponse(event: DiscreteMatrixEvent) {
+    return (
+      this.eventMayTriggerResponse(event) &&
+      event.getType() !== APP_BOXEL_COMMAND_RESULT_EVENT_TYPE
+    );
+  }
 
   constructor(
     readonly client: MatrixClient,
@@ -25,6 +61,7 @@ export class Responder {
 
   messagePromises: Promise<ISendEventResponse | void>[] = [];
 
+  initialMessageSent = false;
   responseEventId: string | undefined;
   latestContent = '';
   toolCalls: ChatCompletionSnapshot.Choice.Message.ToolCall[] = [];
@@ -58,15 +95,18 @@ export class Responder {
     await messagePromise;
   };
 
-  async initialize() {
-    let initialMessage = await sendMessageEvent(
-      this.client,
-      this.roomId,
-      thinkingMessage,
-      undefined,
-      { isStreamingFinished: false },
-    );
-    this.responseEventId = initialMessage.event_id;
+  async ensureThinkingMessageSent() {
+    if (!this.initialMessageSent) {
+      let initialMessage = await sendMessageEvent(
+        this.client,
+        this.roomId,
+        thinkingMessage,
+        undefined,
+        { isStreamingFinished: false },
+      );
+      this.responseEventId = initialMessage.event_id;
+      this.initialMessageSent = true;
+    }
   }
 
   async onChunk(

packages/ai-bot/main.ts

@@ -9,7 +9,6 @@ import {
   isCommandResultStatusApplied,
   getPromptParts,
   extractCardFragmentsFromEvents,
-  eventRequiresResponse,
 } from './helpers';
 
 import {
@@ -188,13 +187,15 @@ Common issues are:
         if (toStartOfTimeline) {
           return; // don't print paginated results
         }
-        if (!eventRequiresResponse(event)) {
-          return; // only print messages
-        }
 
         if (senderMatrixUserId === aiBotUserId) {
           return;
         }
+
+        if (!Responder.eventMayTriggerResponse(event)) {
+          return; // early exit for events that will not trigger a response
+        }
+
         log.info(
           '(%s) (Room: "%s" %s) (Message: %s %s)',
           event.getType(),
@@ -205,7 +206,10 @@ Common issues are:
         );
 
         const responder = new Responder(client, room.roomId);
-        await responder.initialize();
+
+        if (Responder.eventWillDefinitelyTriggerResponse(event)) {
+          await responder.ensureThinkingMessageSent();
+        }
 
         let promptParts: PromptParts;
         let initial = await client.roomInitialSync(room!.roomId, 1000);
@@ -216,6 +220,7 @@ Common issues are:
           if (!promptParts.shouldRespond) {
             return;
           }
+          await responder.ensureThinkingMessageSent();
         } catch (e) {
           log.error(e);
           await responder.onError(

packages/ai-bot/tests/prompt-construction-test.ts

@@ -14,8 +14,8 @@ import {
 import {
   APP_BOXEL_MESSAGE_MSGTYPE,
   APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
-  APP_BOXEL_COMMAND_RESULT_REL_TYPE,
   APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
+  APP_BOXEL_COMMAND_RESULT_REL_TYPE,
   DEFAULT_LLM,
   APP_BOXEL_COMMAND_REQUESTS_KEY,
 } from '@cardstack/runtime-common/matrix-constants';

packages/ai-bot/tests/responding-test.ts

@@ -131,7 +131,7 @@ module('Responding', (hooks) => {
   });
 
   test('Sends thinking message', async () => {
-    await responder.initialize();
+    await responder.ensureThinkingMessageSent();
 
     const sentEvents = fakeMatrixClient.getSentEvents();
     assert.equal(sentEvents.length, 1, 'One event should be sent');
@@ -153,7 +153,7 @@ module('Responding', (hooks) => {
   });
 
   test('Sends first content message immediately, replace the thinking message', async () => {
-    await responder.initialize();
+    await responder.ensureThinkingMessageSent();
 
     // Send several messages
     for (let i = 0; i < 10; i++) {
@@ -188,7 +188,7 @@ module('Responding', (hooks) => {
   });
 
   test('Sends first content message immediately, only sends new content updates after 250ms, replacing the thinking message', async () => {
-    await responder.initialize();
+    await responder.ensureThinkingMessageSent();
 
     // Send several messages
     for (let i = 0; i < 10; i++) {
@@ -260,7 +260,7 @@ module('Responding', (hooks) => {
       },
     };
 
-    await responder.initialize();
+    await responder.ensureThinkingMessageSent();
 
     await responder.onChunk(
       {} as any,
@@ -326,7 +326,7 @@ module('Responding', (hooks) => {
         },
       },
     };
-    await responder.initialize();
+    await responder.ensureThinkingMessageSent();
 
     await responder.onChunk({} as any, snapshotWithContent('some content'));
 
@@ -449,7 +449,7 @@ module('Responding', (hooks) => {
         zipCode: '90210',
       },
     };
-    await responder.initialize();
+    await responder.ensureThinkingMessageSent();
 
     await responder.onChunk({} as any, snapshotWithContent('some content'));