Skip to content

Commit 1353a82

Browse files
lukemelialukemelia
authoredJan 7, 2025
Merge pull request #1947 from cardstack/cs-7598-ai-assistant-chat-should-automatically-display-return-value
Update command results to be the result cards of command executions
2 parents edbebe7 + 8fa0992 commit 1353a82

33 files changed

+1338
-911
lines changed
 

‎packages/ai-bot/helpers.ts

+51-73
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,22 @@ import type {
88
MatrixEvent as DiscreteMatrixEvent,
99
CardFragmentContent,
1010
CommandEvent,
11-
CommandResultEvent,
12-
ReactionEvent,
1311
Tool,
1412
SkillsConfigEvent,
13+
CommandResultEvent,
1514
} from 'https://cardstack.com/base/matrix-event';
1615
import { MatrixEvent, type IRoomEvent } from 'matrix-js-sdk';
1716
import { ChatCompletionMessageToolCall } from 'openai/resources/chat/completions';
1817
import * as Sentry from '@sentry/node';
1918
import { logger } from '@cardstack/runtime-common';
19+
import {
20+
APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
21+
APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
22+
} from '../runtime-common/matrix-constants';
2023
import {
2124
APP_BOXEL_CARDFRAGMENT_MSGTYPE,
2225
APP_BOXEL_MESSAGE_MSGTYPE,
2326
APP_BOXEL_COMMAND_MSGTYPE,
24-
APP_BOXEL_COMMAND_RESULT_MSGTYPE,
2527
APP_BOXEL_ROOM_SKILLS_EVENT_TYPE,
2628
} from '@cardstack/runtime-common/matrix-constants';
2729

@@ -140,6 +142,16 @@ export function constructHistory(
140142
}
141143
}
142144
let event = { ...rawEvent } as DiscreteMatrixEvent;
145+
if (
146+
event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
147+
event.content.msgtype == APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
148+
) {
149+
let { cardEventId } = event.content.data;
150+
event.content.data.card = serializedCardFromFragments(
151+
cardEventId,
152+
cardFragments,
153+
);
154+
}
143155
if (event.type !== 'm.room.message') {
144156
continue;
145157
}
@@ -358,60 +370,20 @@ export function getToolChoice(
358370
return 'auto';
359371
}
360372

361-
export function isCommandResultEvent(
362-
event: DiscreteMatrixEvent,
363-
): event is CommandResultEvent {
364-
return (
365-
event.type === 'm.room.message' &&
366-
typeof event.content === 'object' &&
367-
event.content.msgtype === APP_BOXEL_COMMAND_RESULT_MSGTYPE
368-
);
369-
}
370-
371-
export function isReactionEvent(
372-
event: DiscreteMatrixEvent,
373-
): event is ReactionEvent {
374-
return (
375-
event.type === 'm.reaction' &&
376-
event.content['m.relates_to'].rel_type === 'm.annotation'
377-
);
378-
}
379-
380-
function getReactionStatus(
381-
commandEvent: DiscreteMatrixEvent,
382-
history: DiscreteMatrixEvent[],
383-
) {
384-
let maybeReactionEvent = history.find((e) => {
385-
if (
386-
isReactionEvent(e) &&
387-
e.content['m.relates_to']?.event_id === commandEvent.event_id
388-
) {
389-
return true;
390-
}
391-
return false;
392-
});
393-
return maybeReactionEvent && isReactionEvent(maybeReactionEvent)
394-
? maybeReactionEvent.content['m.relates_to'].key
395-
: undefined;
396-
}
397-
398373
function getCommandResult(
399374
commandEvent: CommandEvent,
400375
history: DiscreteMatrixEvent[],
401376
) {
402-
let maybeCommandResultEvent = history.find((e) => {
377+
let commandResultEvent = history.find((e) => {
403378
if (
404379
isCommandResultEvent(e) &&
405380
e.content['m.relates_to']?.event_id === commandEvent.event_id
406381
) {
407382
return true;
408383
}
409384
return false;
410-
});
411-
return maybeCommandResultEvent &&
412-
isCommandResultEvent(maybeCommandResultEvent)
413-
? maybeCommandResultEvent.content.result
414-
: undefined;
385+
}) as CommandResultEvent | undefined;
386+
return commandResultEvent;
415387
}
416388

417389
function toToolCall(event: CommandEvent): ChatCompletionMessageToolCall {
@@ -429,21 +401,26 @@ function toPromptMessageWithToolResult(
429401
event: CommandEvent,
430402
history: DiscreteMatrixEvent[],
431403
): OpenAIPromptMessage {
432-
let commandResult = getCommandResult(event as CommandEvent, history);
404+
let commandResult = getCommandResult(event, history);
405+
let content = 'pending';
433406
if (commandResult) {
434-
return {
435-
role: 'tool',
436-
content: commandResult,
437-
tool_call_id: event.content.data.toolCall.id,
438-
};
439-
} else {
440-
let reactionStatus = getReactionStatus(event, history);
441-
return {
442-
role: 'tool',
443-
content: reactionStatus ?? 'pending',
444-
tool_call_id: event.content.data.toolCall.id,
445-
};
407+
let status = commandResult.content['m.relates_to']?.key;
408+
if (
409+
commandResult.content.msgtype ===
410+
APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
411+
) {
412+
content = `Command ${status}, with result card: ${JSON.stringify(
413+
commandResult.content.data.card,
414+
)}.\n`;
415+
} else {
416+
content = `Command ${status}.\n`;
417+
}
446418
}
419+
return {
420+
role: 'tool',
421+
content,
422+
tool_call_id: event.content.data.toolCall.id,
423+
};
447424
}
448425

449426
export function getModifyPrompt(
@@ -570,24 +547,13 @@ export function cleanContent(content: string) {
570547
return content.trim();
571548
}
572549

573-
export const isCommandReactionEvent = (event?: MatrixEvent) => {
574-
if (event === undefined) {
575-
return false;
576-
}
577-
let content = event.getContent();
578-
return (
579-
event.getType() === 'm.reaction' &&
580-
content['m.relates_to']?.rel_type === 'm.annotation'
581-
);
582-
};
583-
584-
export const isCommandReactionStatusApplied = (event?: MatrixEvent) => {
550+
export const isCommandResultStatusApplied = (event?: MatrixEvent) => {
585551
if (event === undefined) {
586552
return false;
587553
}
588-
let content = event.getContent();
589554
return (
590-
isCommandReactionEvent(event) && content['m.relates_to']?.key === 'applied'
555+
isCommandResultEvent(event.event as DiscreteMatrixEvent) &&
556+
event.getContent()['m.relates_to']?.key === 'applied'
591557
);
592558
};
593559

@@ -603,3 +569,15 @@ export function isCommandEvent(
603569
typeof event.content.data.toolCall === 'object'
604570
);
605571
}
572+
573+
export function isCommandResultEvent(
574+
event?: DiscreteMatrixEvent,
575+
): event is CommandResultEvent {
576+
if (event === undefined) {
577+
return false;
578+
}
579+
return (
580+
event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
581+
event.content['m.relates_to']?.rel_type === 'm.annotation'
582+
);
583+
}

‎packages/ai-bot/lib/set-title.ts

+14-9
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,23 @@
1-
import { type MatrixEvent, type IEventRelation } from 'matrix-js-sdk';
1+
import {
2+
type MatrixEvent,
3+
type IEventRelation,
4+
IRoomEvent,
5+
} from 'matrix-js-sdk';
26
import OpenAI from 'openai';
37
import {
48
type OpenAIPromptMessage,
5-
isCommandReactionStatusApplied,
9+
isCommandResultStatusApplied,
610
attachedCardsToMessage,
711
isCommandEvent,
812
getRelevantCards,
913
} from '../helpers';
1014
import { MatrixClient } from './matrix';
1115
import type { MatrixEvent as DiscreteMatrixEvent } from 'https://cardstack.com/base/matrix-event';
16+
import { ChatCompletionMessageParam } from 'openai/resources';
1217

1318
const SET_TITLE_SYSTEM_MESSAGE = `You are a chat titling system, you must read the conversation and return a suggested title of no more than six words.
14-
Do NOT say talk or discussion or discussing or chat or chatting, this is implied by the context.
15-
The user can optionally apply 'patchCard' by sending data about fields to update.
19+
Do NOT say talk or discussion or discussing or chat or chatting, this is implied by the context.
20+
The user can optionally apply 'patchCard' by sending data about fields to update.
1621
Explain the general actions and user intent. If 'patchCard' was used, express the title in an active sentence. Do NOT use the word "patch" in the title.`;
1722

1823
export async function setTitle(
@@ -39,7 +44,7 @@ export async function setTitle(
3944
let result = await openai.chat.completions.create(
4045
{
4146
model: 'gpt-4o',
42-
messages: startOfConversation,
47+
messages: startOfConversation as ChatCompletionMessageParam[],
4348
stream: false,
4449
},
4550
{
@@ -120,15 +125,15 @@ export const getLatestCommandApplyMessage = (
120125
return [];
121126
};
122127

123-
export const roomTitleAlreadySet = (rawEventLog: DiscreteMatrixEvent[]) => {
128+
export const roomTitleAlreadySet = (rawEventLog: IRoomEvent[]) => {
124129
return (
125130
rawEventLog.filter((event) => event.type === 'm.room.name').length > 1 ??
126131
false
127132
);
128133
};
129134

130135
const userAlreadyHasSentNMessages = (
131-
rawEventLog: DiscreteMatrixEvent[],
136+
rawEventLog: IRoomEvent[],
132137
botUserId: string,
133138
n = 5,
134139
) => {
@@ -140,12 +145,12 @@ const userAlreadyHasSentNMessages = (
140145
};
141146

142147
export function shouldSetRoomTitle(
143-
rawEventLog: DiscreteMatrixEvent[],
148+
rawEventLog: IRoomEvent[],
144149
aiBotUserId: string,
145150
event?: MatrixEvent,
146151
) {
147152
return (
148-
(isCommandReactionStatusApplied(event) ||
153+
(isCommandResultStatusApplied(event) ||
149154
userAlreadyHasSentNMessages(rawEventLog, aiBotUserId)) &&
150155
!roomTitleAlreadySet(rawEventLog)
151156
);

‎packages/ai-bot/main.ts

+7-5
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import { logger, aiBotUsername } from '@cardstack/runtime-common';
1111
import {
1212
type PromptParts,
1313
constructHistory,
14-
isCommandReactionStatusApplied,
14+
isCommandResultStatusApplied,
1515
getPromptParts,
1616
extractCardFragmentsFromEvents,
1717
} from './helpers';
@@ -30,6 +30,8 @@ import * as Sentry from '@sentry/node';
3030

3131
import { getAvailableCredits, saveUsageCost } from './lib/ai-billing';
3232
import { PgAdapter } from '@cardstack/postgres';
33+
import { ChatCompletionMessageParam } from 'openai/resources';
34+
import { OpenAIError } from 'openai/error';
3335

3436
let log = logger('ai-bot');
3537

@@ -69,12 +71,12 @@ class Assistant {
6971
if (prompt.tools.length === 0) {
7072
return this.openai.beta.chat.completions.stream({
7173
model: prompt.model,
72-
messages: prompt.messages,
74+
messages: prompt.messages as ChatCompletionMessageParam[],
7375
});
7476
} else {
7577
return this.openai.beta.chat.completions.stream({
7678
model: prompt.model,
77-
messages: prompt.messages,
79+
messages: prompt.messages as ChatCompletionMessageParam[],
7880
tools: prompt.tools,
7981
tool_choice: prompt.toolChoice,
8082
});
@@ -250,7 +252,7 @@ Common issues are:
250252
finalContent = await runner.finalContent();
251253
await responder.finalize(finalContent);
252254
} catch (error) {
253-
await responder.onError(error);
255+
await responder.onError(error as OpenAIError);
254256
} finally {
255257
if (generationId) {
256258
assistant.trackAiUsageCost(senderMatrixUserId, generationId);
@@ -278,7 +280,7 @@ Common issues are:
278280
if (!room) {
279281
return;
280282
}
281-
if (!isCommandReactionStatusApplied(event)) {
283+
if (!isCommandResultStatusApplied(event)) {
282284
return;
283285
}
284286
log.info(

‎packages/ai-bot/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
},
1919
"devDependencies": {
2020
"@sinonjs/fake-timers": "^11.2.2",
21+
"@types/qunit": "^2.19.12",
2122
"@types/sinonjs__fake-timers": "^8.1.5",
2223
"qunit": "^2.18.0"
2324
},

0 commit comments

Comments
 (0)