Show ApplyButton's preparing state when a command is being prepared

Author: lukemelia

packages/ai-bot/lib/matrix.ts

@@ -82,10 +82,11 @@ export async function sendMessageEvent(
 export async function sendCommandEvent(
   client: MatrixClient,
   roomId: string,
+  messageBody: string,
   functionCall: FunctionToolCall,
   eventToUpdate: string | undefined,
 ) {
-  let messageObject = toMatrixMessageCommandContent(functionCall);
+  let messageObject = toMatrixMessageCommandContent(functionCall, messageBody);
 
   if (messageObject !== undefined) {
     return await sendMatrixEvent(
@@ -128,9 +129,10 @@ export async function sendErrorEvent(
 
 export const toMatrixMessageCommandContent = (
   functionCall: FunctionToolCall,
+  messageBody: string | undefined,
 ): IContent | undefined => {
   let { arguments: payload } = functionCall;
-  const body = payload['description'] || 'Issuing command';
+  const body = messageBody || payload['description'] || 'Issuing command';
   let messageObject: IContent = {
     body: body,
     msgtype: APP_BOXEL_COMMAND_MSGTYPE,

packages/ai-bot/lib/responder.ts

@@ -124,15 +124,24 @@ export class Responder {
     for (const toolCall of msg.tool_calls || []) {
       log.debug('[Room Timeline] Function call', toolCall);
       try {
+        const functionToolCall = this.deserializeToolCall(toolCall);
+        this.latestContent = [
+          this.latestContent,
+          functionToolCall.arguments['description'],
+        ]
+          .filter(Boolean)
+          .join('\n\n');
         let commandEventPromise = sendCommandEvent(
           this.client,
           this.roomId,
-          this.deserializeToolCall(toolCall),
-          this.initialMessageReplaced ? undefined : this.responseEventId,
+          this.latestContent,
+          functionToolCall,
+          this.responseEventId,
         );
         this.messagePromises.push(commandEventPromise);
         await commandEventPromise;
         this.initialMessageReplaced = true;
+        this.isStreamingFinished = true;
       } catch (error) {
         Sentry.captureException(error);
         this.initialMessageReplaced = true;
@@ -159,7 +168,7 @@ export class Responder {
   }
 
   async finalize(finalContent: string | void | null | undefined) {
-    if (finalContent) {
+    if (finalContent && !this.isStreamingFinished) {
       this.latestContent = cleanContent(finalContent);
       this.isStreamingFinished = true;
       await this.sendMessageEventWithDebouncing();

packages/ai-bot/tests/responding-test.ts

@@ -269,7 +269,7 @@ module('Responding', (hooks) => {
     );
   });
 
-  test('Sends tool call event separately when content is sent before tool call', async () => {
+  test('Sends tool call event and adds to event when content is sent before tool call', async () => {
     const patchArgs = {
       description: 'A new thing',
       attributes: {
@@ -303,13 +303,37 @@ module('Responding', (hooks) => {
     assert.equal(
       sentEvents.length,
       3,
-      'Thinking message, and tool call event should be sent',
+      'Thinking message, and content, and tool call event should be sent',
     );
     assert.equal(
       sentEvents[0].content.body,
       thinkingMessage,
       'Thinking message should be sent first',
     );
+    assert.notOk(
+      sentEvents[0].content['m.relates_to'],
+      'The tool call event should not replace any message',
+    );
+    assert.equal(
+      sentEvents[1].content.body,
+      'some content',
+      'Content message should be sent next',
+    );
+    assert.strictEqual(
+      sentEvents[1].content['m.relates_to']?.event_id,
+      sentEvents[0].eventId,
+      'The content event should replace the initial message',
+    );
+    assert.equal(
+      sentEvents[2].content.body,
+      'some content\n\nA new thing',
+      'Content message plus function description should be sent next',
+    );
+    assert.strictEqual(
+      sentEvents[2].content['m.relates_to']?.event_id,
+      sentEvents[0].eventId,
+      'The command event should replace the initial message',
+    );
     assert.deepEqual(
       JSON.parse(sentEvents[2].content.data),
       {
@@ -332,24 +356,6 @@ module('Responding', (hooks) => {
       },
       'Tool call event should be sent with correct content',
     );
-    assert.notOk(
-      sentEvents[2].content['m.relates_to'],
-      'The tool call event should not replace any message',
-    );
-
-    assert.equal(
-      sentEvents[1].content.body,
-      'some content',
-      'Content event should be sent',
-    );
-    assert.deepEqual(
-      sentEvents[1].content['m.relates_to'],
-      {
-        rel_type: 'm.replace',
-        event_id: '0',
-      },
-      'The content event should replace the thinking message',
-    );
   });
 
   test('Updates message type to command when tool call is in progress', async () => {

packages/host/app/components/ai-assistant/apply-button/index.gts

@@ -44,6 +44,7 @@ const AiAssistantApplyButton: TemplateOnlyComponent<Signature> = <template>
           @kind='secondary-dark'
           @size='small'
           class='apply-button'
+          tabindex='-1'
           {{setCssVar boxel-button-text-color='var(--boxel-200)'}}
           data-test-apply-state='preparing'
           ...attributes
@@ -93,6 +94,12 @@ const AiAssistantApplyButton: TemplateOnlyComponent<Signature> = <template>
       border: 0;
       min-width: 74px;
     }
+    .state-indicator.preparing .apply-button:hover,
+    .state-indicator.preparing .apply-button:focus {
+      --boxel-button-color: inherit;
+      filter: none;
+      cursor: not-allowed;
+    }
 
     .state-indicator.preparing::before {
       content: '';
@@ -150,6 +157,12 @@ const AiAssistantApplyButton: TemplateOnlyComponent<Signature> = <template>
       background-color: var(--boxel-error-200);
       border-color: var(--boxel-error-200);
     }
+
+    @keyframes spin {
+      to {
+        transform: rotate(360deg);
+      }
+    }
   </style>
 </template>;
 

packages/host/app/components/ai-assistant/message/index.gts

@@ -41,6 +41,7 @@ interface Signature {
     }) => void;
     errorMessage?: string;
     isPending?: boolean;
+    isCommandMessage?: boolean;
     retryAction?: () => void;
   };
   Blocks: { default: [] };
@@ -170,7 +171,11 @@ export default class AiAssistantMessage extends Component<Signature> {
           </div>
         {{/if}}
 
-        <div class='content' data-test-ai-message-content>
+        <div
+          class='content'
+          data-test-ai-message-content
+          data-test-command-message={{@isCommandMessage}}
+        >
           {{@formattedMessage}}
 
           {{yield}}

packages/host/app/components/ai-assistant/message/usage.gts

@@ -15,6 +15,7 @@ export default class AiAssistantMessageUsage extends Component {
   @tracked datetime = new Date(2024, 0, 3, 12, 30);
   @tracked isFromAssistant = false;
   @tracked isStreaming = false;
+  @tracked isCommandMessage = false;
   @tracked userId = 'johndoe:boxel.ai';
   @tracked errorMessage = '';
 
@@ -63,6 +64,7 @@ export default class AiAssistantMessageUsage extends Component {
               @errorMessage={{this.errorMessage}}
               @retryAction={{this.retryAction}}
               @isStreaming={{this.isStreaming}}
+              @isCommandMessage={{this.isCommandMessage}}
             >
               <em>Optional embedded content</em>
             </AiAssistantMessage>
@@ -136,6 +138,7 @@ export default class AiAssistantMessageUsage extends Component {
                 isReady=true
               }}
               @isStreaming={{false}}
+              @isCommandMessage={{false}}
             />
             <AiAssistantMessage
               @formattedMessage={{htmlSafe
@@ -146,6 +149,7 @@ export default class AiAssistantMessageUsage extends Component {
               @datetime={{this.oneMinutesAgo}}
               @isFromAssistant={{true}}
               @isStreaming={{false}}
+              @isCommandMessage={{false}}
             />
           </AiAssistantConversation>
         </div>

packages/host/app/components/matrix/preparing-room-message-command.gts

@@ -0,0 +1,28 @@
+import { TemplateOnlyComponent } from '@ember/component/template-only';
+
+import ApplyButton from '../ai-assistant/apply-button';
+
+interface Signature {
+  Element: HTMLDivElement;
+}
+
+const RoomMessageCommand: TemplateOnlyComponent<Signature> = <template>
+  <div ...attributes>
+    <div class='command-button-bar'>
+      <ApplyButton @state='preparing' data-test-command-apply='preparing' />
+    </div>
+  </div>
+
+  {{! template-lint-disable no-whitespace-for-layout  }}
+  {{! ignore the above error because ember-template-lint complains about the whitespace in the multi-line comment below }}
+  <style scoped>
+    .command-button-bar {
+      display: flex;
+      justify-content: flex-end;
+      gap: var(--boxel-sp-xs);
+      margin-top: var(--boxel-sp);
+    }
+  </style>
+</template>;
+
+export default RoomMessageCommand;

packages/host/app/components/matrix/room-message.gts

@@ -10,7 +10,7 @@ import { trackedFunction } from 'ember-resources/util/function';
 
 import { Avatar } from '@cardstack/boxel-ui/components';
 
-import { bool } from '@cardstack/boxel-ui/helpers';
+import { bool, or } from '@cardstack/boxel-ui/helpers';
 
 import { markdownToHtml } from '@cardstack/runtime-common';
 
@@ -25,6 +25,7 @@ import { type CardDef } from 'https://cardstack.com/base/card-api';
 import AiAssistantMessage from '../ai-assistant/message';
 import { aiBotUserId } from '../ai-assistant/panel';
 
+import PreparingRoomMessageCommand from './preparing-room-message-command';
 import RoomMessageCommand from './room-message-command';
 
 interface Signature {
@@ -119,6 +120,10 @@ export default class RoomMessage extends Component<Signature> {
         @isStreaming={{@isStreaming}}
         @retryAction={{if @message.command (perform this.run) @retryAction}}
         @isPending={{@isPending}}
+        @isCommandMessage={{or
+          (bool @message.command)
+          @message.isPreparingCommand
+        }}
         data-test-boxel-message-from={{@message.author.name}}
         data-test-boxel-message-instance-id={{@message.instanceId}}
         ...attributes
@@ -137,6 +142,8 @@ export default class RoomMessage extends Component<Signature> {
             @failedCommandState={{this.failedCommandState}}
             @isError={{bool this.errorMessage}}
           />
+        {{else if @message.isPreparingCommand}}
+          <PreparingRoomMessageCommand />
         {{/if}}
       </AiAssistantMessage>
     {{/if}}

packages/host/app/lib/matrix-classes/message-builder.ts

@@ -129,13 +129,15 @@ export default class MessageBuilder {
       message.attachedCardIds = this.attachedCardIds;
     } else if (event.content.msgtype === 'm.text') {
       message.isStreamingFinished = !!event.content.isStreamingFinished; // Indicates whether streaming (message updating while AI bot is sending more content into the message) has finished
-    } else if (
-      event.content.msgtype === APP_BOXEL_COMMAND_MSGTYPE &&
-      event.content.data.toolCall
-    ) {
+    } else if (event.content.msgtype === APP_BOXEL_COMMAND_MSGTYPE) {
       message.formattedMessage = this.formattedMessageForCommand;
-      message.command = this.buildMessageCommand(message);
-      message.isStreamingFinished = true;
+      const command = this.buildMessageCommand(message);
+      if (command) {
+        message.command = command;
+      } else {
+        message.isPreparingCommand = true;
+      }
+      message.isStreamingFinished = !!event.content.isStreamingFinished; // Indicates whether streaming (message updating while AI bot is sending more content into the message) has finished
     }
     return message;
   }
@@ -159,16 +161,16 @@ export default class MessageBuilder {
     message.updated = new Date();
 
     if (this.event.content.msgtype === APP_BOXEL_COMMAND_MSGTYPE) {
-      if (!message.command) {
-        message.command = this.buildMessageCommand(message);
+      const latestCommand = this.buildMessageCommand(message);
+      if (!message.command && latestCommand) {
+        message.command = latestCommand;
+        message.isPreparingCommand = false;
+      } else if (latestCommand && message.command) {
+        message.command.name = latestCommand.name;
+        message.command.payload = latestCommand.payload;
+      } else {
+        message.isPreparingCommand = true;
       }
-
-      message.isStreamingFinished = true;
-      message.formattedMessage = this.formattedMessageForCommand;
-
-      let command = this.event.content.data.toolCall;
-      message.command.name = command.name;
-      message.command.payload = command.arguments;
     }
   }
 
@@ -177,7 +179,7 @@ export default class MessageBuilder {
       message.command = this.buildMessageCommand(message);
     }
 
-    if (this.builderContext.commandResultEvent) {
+    if (this.builderContext.commandResultEvent && message.command) {
       let event = this.builderContext.commandResultEvent;
       message.command.commandStatus = event.content['m.relates_to']
         .key as CommandStatus;
@@ -197,27 +199,29 @@ export default class MessageBuilder {
         return (
           e.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
           r.rel_type === 'm.annotation' &&
-          (r.event_id === event!.content.data.eventId ||
-            r.event_id === event!.event_id ||
+          (r.event_id === event!.event_id ||
             r.event_id === this.builderContext.effectiveEventId)
         );
       }) as CommandResultEvent | undefined);
 
-    let command = event.content.data.toolCall;
-    let messageCommand = new MessageCommand(
-      message,
-      command.id,
-      command.name,
-      command.arguments,
-      this.builderContext.effectiveEventId,
-      (commandResultEvent?.content['m.relates_to']?.key ||
-        'ready') as CommandStatus,
-      commandResultEvent?.content.msgtype ===
-      APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
-        ? commandResultEvent.content.data.cardEventId
-        : undefined,
-      getOwner(this)!,
-    );
-    return messageCommand;
+    if (event.content.isStreamingFinished !== false && event.content.data) {
+      let command = event.content.data.toolCall;
+      let messageCommand = new MessageCommand(
+        message,
+        command.id,
+        command.name,
+        command.arguments,
+        this.builderContext.effectiveEventId,
+        (commandResultEvent?.content['m.relates_to']?.key ||
+          'ready') as CommandStatus,
+        commandResultEvent?.content.msgtype ===
+        APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
+          ? commandResultEvent.content.data.cardEventId
+          : undefined,
+        getOwner(this)!,
+      );
+      return messageCommand;
+    }
+    return null;
   }
 }

packages/host/app/lib/matrix-classes/message.ts

@@ -47,6 +47,7 @@ export class Message implements RoomMessageInterface {
   @tracked formattedMessage: string;
   @tracked message: string;
   @tracked command?: MessageCommand | null;
+  @tracked isPreparingCommand?: boolean;
   @tracked isStreamingFinished?: boolean;
 
   attachedCardIds?: string[] | null;