Capture reasoning

Author: lukemelia

packages/ai-bot/lib/debug.ts

@@ -61,6 +61,7 @@ export async function handleDebugCommands(
         client,
         roomId,
         `Error parsing your debug patch, ${error} ${patchMessage}`,
+        '',
         undefined,
       );
     }

packages/ai-bot/lib/matrix.ts

@@ -3,7 +3,10 @@ import { logger } from '@cardstack/runtime-common';
 import { OpenAIError } from 'openai/error';
 import * as Sentry from '@sentry/node';
 import { FunctionToolCall } from '@cardstack/runtime-common/helpers/ai';
-import { APP_BOXEL_COMMAND_MSGTYPE } from '@cardstack/runtime-common/matrix-constants';
+import {
+  APP_BOXEL_COMMAND_MSGTYPE,
+  APP_BOXEL_REASONING_CONTENT_KEY,
+} from '@cardstack/runtime-common/matrix-constants';
 
 let log = logger('ai-bot');
 
@@ -56,21 +59,24 @@ export async function updateStateEvent(
 export async function sendMessage(
   client: MatrixClient,
   roomId: string,
-  content: string,
+  body: string,
+  reasoning: string,
   eventToUpdate: string | undefined,
   data: any = {},
 ) {
-  log.debug('sending message', content);
+  log.debug('sending message', body);
   let messageObject: IContent = {
     ...{
-      body: content,
+      body,
       msgtype: 'm.text',
-      formatted_body: content,
+      formatted_body: body,
       format: 'org.matrix.custom.html',
+      [APP_BOXEL_REASONING_CONTENT_KEY]: reasoning,
       'm.new_content': {
-        body: content,
+        body,
         msgtype: 'm.text',
-        formatted_body: content,
+        formatted_body: body,
+        [APP_BOXEL_REASONING_CONTENT_KEY]: reasoning,
         format: 'org.matrix.custom.html',
       },
     },
@@ -121,6 +127,7 @@ export async function sendError(
       client,
       roomId,
       'There was an error processing your request, please try again later',
+      '',
       eventToUpdate,
       {
         isStreamingFinished: true,

packages/ai-bot/lib/responder.ts

@@ -22,31 +22,29 @@ let log = logger('ai-bot');
 export class Responder {
   // internally has a debounced function that will send the text messages
 
-  initialMessageId: string | undefined;
+  responseEventId: string | undefined;
   initialMessageReplaced = false;
   client: MatrixClient;
   roomId: string;
   includesFunctionToolCall = false;
-  latestContent?: string;
+  latestContent = '';
+  reasoning = '';
   messagePromises: Promise<ISendEventResponse | void>[] = [];
-  debouncedMessageSender: (
-    content: string,
-    eventToUpdate: string | undefined,
-    isStreamingFinished?: boolean,
-  ) => Promise<void>;
+  isStreamingFinished = false;
+  sendMessageDebounced: () => Promise<void>;
 
   constructor(client: MatrixClient, roomId: string) {
     this.roomId = roomId;
     this.client = client;
-    this.debouncedMessageSender = debounce(
-      async (
-        content: string,
-        eventToUpdate: string | undefined,
-        isStreamingFinished = false,
-      ) => {
-        this.latestContent = content;
+    this.sendMessageDebounced = debounce(
+      async () => {
+        const content = this.latestContent;
+        const reasoning = this.reasoning;
+        const eventToUpdate = this.responseEventId;
+        const isStreamingFinished = this.isStreamingFinished;
+
         let dataOverrides: Record<string, string | boolean> = {
-          isStreamingFinished: isStreamingFinished,
+          isStreamingFinished,
         };
         if (this.includesFunctionToolCall) {
           dataOverrides = {
@@ -58,6 +56,7 @@ export class Responder {
           this.client,
           this.roomId,
           content,
+          reasoning,
           eventToUpdate,
           dataOverrides,
         );
@@ -74,23 +73,26 @@ export class Responder {
       this.client,
       this.roomId,
       thinkingMessage,
+      '',
       undefined,
       { isStreamingFinished: false },
     );
-    this.initialMessageId = initialMessage.event_id;
+    this.responseEventId = initialMessage.event_id;
   }
 
   async onChunk(chunk: OpenAI.Chat.Completions.ChatCompletionChunk) {
     log.debug('onChunk: ', JSON.stringify(chunk, null, 2));
     if (chunk.choices[0].delta?.tool_calls?.[0]?.function) {
       if (!this.includesFunctionToolCall) {
         this.includesFunctionToolCall = true;
-        await this.debouncedMessageSender(
-          this.latestContent || '',
-          this.initialMessageId,
-        );
+        await this.sendMessageDebounced();
       }
     }
+    // @ts-expect-error reasoning is not in the types yet
+    if (chunk.choices[0].delta?.reasoning) {
+      // @ts-expect-error reasoning is not in the types yet
+      this.reasoning += chunk.choices[0].delta.reasoning;
+    }
     // This usage value is set *once* and *only once* at the end of the conversation
     // It will be null at all other times.
     if (chunk.usage) {
@@ -102,10 +104,8 @@ export class Responder {
 
   async onContent(snapshot: string) {
     log.debug('onContent: ', snapshot);
-    await this.debouncedMessageSender(
-      cleanContent(snapshot),
-      this.initialMessageId,
-    );
+    this.latestContent = cleanContent(snapshot);
+    await this.sendMessageDebounced();
     this.initialMessageReplaced = true;
   }
 
@@ -142,7 +142,7 @@ export class Responder {
           this.client,
           this.roomId,
           this.deserializeToolCall(toolCall),
-          this.initialMessageId,
+          this.responseEventId,
         );
         this.messagePromises.push(commandMessagePromise);
         await commandMessagePromise;
@@ -154,7 +154,7 @@ export class Responder {
           this.client,
           this.roomId,
           error,
-          this.initialMessageId,
+          this.responseEventId,
         );
         this.messagePromises.push(errorPromise);
         await errorPromise;
@@ -169,19 +169,16 @@ export class Responder {
       this.client,
       this.roomId,
       error,
-      this.initialMessageId,
+      this.responseEventId,
     );
   }
 
   async finalize(finalContent: string | void | null | undefined) {
     log.debug('finalize: ', finalContent);
     if (finalContent) {
-      finalContent = cleanContent(finalContent);
-      await this.debouncedMessageSender(
-        finalContent,
-        this.initialMessageId,
-        true,
-      );
+      this.latestContent = cleanContent(finalContent);
+      this.isStreamingFinished = true;
+      await this.sendMessageDebounced();
     }
     await Promise.all(this.messagePromises);
   }

packages/ai-bot/main.ts

@@ -97,13 +97,15 @@ class Assistant {
       return this.openai.beta.chat.completions.stream({
         model: prompt.model,
         messages: prompt.messages as ChatCompletionMessageParam[],
+        include_reasoning: true,
       });
     } else {
       return this.openai.beta.chat.completions.stream({
         model: prompt.model,
         messages: prompt.messages as ChatCompletionMessageParam[],
         tools: prompt.tools,
         tool_choice: prompt.toolChoice,
+        include_reasoning: true,
       });
     }
   }

packages/ai-bot/package.json

@@ -5,11 +5,11 @@
     "@cardstack/postgres": "workspace:*",
     "@cardstack/billing": "workspace:*",
     "@sentry/node": "^8.31.0",
-    "@types/node": "^18.18.5",
+    "@types/node": "^20.17.16",
     "@types/stream-chain": "^2.0.1",
     "@types/stream-json": "^1.7.3",
     "matrix-js-sdk": "^31.0.0",
-    "openai": "4.47.1",
+    "openai": "4.81.0",
     "qunit": "^2.18.0",
     "stream-chain": "^2.2.5",
     "stream-json": "^1.8.0",

packages/base/matrix-event.gts

@@ -6,15 +6,16 @@ import {
   type ToolChoice,
 } from '@cardstack/runtime-common/helpers/ai';
 import {
+  APP_BOXEL_ACTIVE_LLM,
   APP_BOXEL_CARD_FORMAT,
   APP_BOXEL_CARDFRAGMENT_MSGTYPE,
   APP_BOXEL_COMMAND_MSGTYPE,
   APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
   APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE,
   APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
   APP_BOXEL_MESSAGE_MSGTYPE,
+  APP_BOXEL_REASONING_CONTENT_KEY,
   APP_BOXEL_ROOM_SKILLS_EVENT_TYPE,
-  APP_BOXEL_ACTIVE_LLM,
 } from '@cardstack/runtime-common/matrix-constants';
 
 interface BaseMatrixEvent {
@@ -112,6 +113,7 @@ export interface MessageEvent extends BaseMatrixEvent {
     format: 'org.matrix.custom.html';
     body: string;
     formatted_body: string;
+    [APP_BOXEL_REASONING_CONTENT_KEY]?: string;
     isStreamingFinished: boolean;
     errorMessage?: string;
   };

packages/runtime-common/matrix-constants.ts

@@ -9,6 +9,7 @@ export const APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE =
   'app.boxel.commandResultWithNoOutput';
 export const APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE =
   'app.boxel.realm-server-event';
+export const APP_BOXEL_REASONING_CONTENT_KEY = 'app.boxel.reasoning';
 export const APP_BOXEL_ROOM_SKILLS_EVENT_TYPE = 'app.boxel.room.skills';
 export const APP_BOXEL_REALMS_EVENT_TYPE = 'app.boxel.realms';
 export const LEGACY_APP_BOXEL_REALMS_EVENT_TYPE = 'com.cardstack.boxel.realms';