Merge pull request #2023 from cardstack/run-ai-on-response

Attach tool call results to the call & trigger responses on tool calls

Author: IanCal

packages/ai-bot/helpers.ts

@@ -156,7 +156,10 @@ export function constructHistory(
         cardFragments,
       );
     }
-    if (event.type !== 'm.room.message') {
+    if (
+      event.type !== 'm.room.message' &&
+      event.type !== APP_BOXEL_COMMAND_RESULT_EVENT_TYPE
+    ) {
       continue;
     }
     let eventId = event.event_id!;
@@ -595,3 +598,24 @@ export function isCommandResultEvent(
     event.content['m.relates_to']?.rel_type === 'm.annotation'
   );
 }
+
+export function eventRequiresResponse(event: MatrixEvent) {
+  // If it's a message, we should respond unless it's a card fragment
+  if (event.getType() === 'm.room.message') {
+    if (event.getContent().msgtype === APP_BOXEL_CARDFRAGMENT_MSGTYPE) {
+      return false;
+    }
+    return true;
+  }
+
+  // If it's a command result with output, we should respond
+  if (
+    event.getType() === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
+    event.getContent().msgtype === APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
+  ) {
+    return true;
+  }
+
+  // If it's a different type, or a command result without output, we should not respond
+  return false;
+}

packages/ai-bot/main.ts

@@ -14,9 +14,9 @@ import {
   isCommandResultStatusApplied,
   getPromptParts,
   extractCardFragmentsFromEvents,
+  eventRequiresResponse,
 } from './helpers';
 import {
-  APP_BOXEL_CARDFRAGMENT_MSGTYPE,
   APP_BOXEL_ACTIVE_LLM,
   DEFAULT_LLM,
 } from '@cardstack/runtime-common/matrix-constants';
@@ -179,12 +179,9 @@ Common issues are:
         if (toStartOfTimeline) {
           return; // don't print paginated results
         }
-        if (event.getType() !== 'm.room.message') {
+        if (!eventRequiresResponse(event)) {
           return; // only print messages
         }
-        if (event.getContent().msgtype === APP_BOXEL_CARDFRAGMENT_MSGTYPE) {
-          return; // don't respond to card fragments, we just gather these in our history
-        }
 
         if (senderMatrixUserId === aiBotUserId) {
           return;

packages/ai-bot/tests/prompt-construction-test.ts

@@ -1611,6 +1611,27 @@ test('Tools can be required to be called if done so in the last message', () =>
   });
 });
 
+test('Tools calls are connected to their results', () => {
+  const eventList: DiscreteMatrixEvent[] = JSON.parse(
+    readFileSync(
+      path.join(
+        __dirname,
+        'resources/chats/connect-tool-calls-to-results.json',
+      ),
+    ),
+  );
+
+  const { messages } = getPromptParts(eventList, '@aibot:localhost');
+  // find the message with the tool call and its id
+  // it should have the result deserialised
+  const toolCallMessage = messages.find((message) => message.role === 'tool');
+  assert.ok(toolCallMessage, 'Should have a tool call message');
+  assert.ok(
+    toolCallMessage!.content!.includes('Cloudy'),
+    'Tool call result should include "Cloudy"',
+  );
+});
+
 module('set model in prompt', () => {
   test('default active LLM must be equal to `DEFAULT_LLM`', () => {
     const eventList: DiscreteMatrixEvent[] = JSON.parse(