Merge pull request #2089 from cardstack/support-non-tool-call-models

Support non tool call models

Author: IanCal

Diff for: packages/ai-bot/main.ts

@@ -46,6 +46,7 @@ const MINIMUM_CREDITS = 10;
 class Assistant {
   private openai: OpenAI;
   private client: MatrixClient;
+  private toolCallCapableModels: Set<string>;
   pgAdapter: PgAdapter;
   id: string;
 
@@ -57,6 +58,21 @@ class Assistant {
     this.id = id;
     this.client = client;
     this.pgAdapter = new PgAdapter();
+    this.toolCallCapableModels = new Set();
+  }
+
+  async loadToolCallCapableModels() {
+    // api request is https://openrouter.ai/api/v1/models?supported_parameters=tools
+    let response = await fetch(
+      'https://openrouter.ai/api/v1/models?supported_parameters=tools',
+    );
+    let responseJson = (await response.json()) as {
+      data: { id: string }[];
+    };
+    let modelList = responseJson.data;
+    this.toolCallCapableModels = new Set(
+      modelList.map((model: any) => model.id),
+    );
   }
 
   async trackAiUsageCost(matrixUserId: string, generationId: string) {
@@ -72,7 +88,12 @@ class Assistant {
   }
 
   getResponse(prompt: PromptParts) {
-    if (prompt.tools.length === 0) {
+    // Sending tools to models that don't support them results in an error
+    // from openrouter.
+    if (
+      prompt.tools.length === 0 ||
+      !this.toolCallCapableModels.has(prompt.model)
+    ) {
       return this.openai.beta.chat.completions.stream({
         model: prompt.model,
         messages: prompt.messages as ChatCompletionMessageParam[],
@@ -144,6 +165,7 @@ Common issues are:
     });
   let { user_id: aiBotUserId } = auth;
   let assistant = new Assistant(client, aiBotUserId);
+  await assistant.loadToolCallCapableModels();
 
   client.on(RoomMemberEvent.Membership, function (_event, member) {
     if (member.membership === 'invite' && member.userId === aiBotUserId) {

Diff for: packages/host/tests/acceptance/ai-assistant-test.gts

@@ -9,6 +9,7 @@ import { baseRealm } from '@cardstack/runtime-common';
 import {
   APP_BOXEL_ACTIVE_LLM,
   DEFAULT_LLM,
+  DEFAULT_LLM_LIST,
 } from '@cardstack/runtime-common/matrix-constants';
 
 import {
@@ -316,7 +317,9 @@ module('Acceptance | AI Assistant tests', function (hooks) {
       .hasText(DEFAULT_LLM.split('/')[1]);
     await click('[data-test-llm-select-selected]');
 
-    assert.dom('[data-test-llm-select-item]').exists({ count: 4 });
+    assert.dom('[data-test-llm-select-item]').exists({
+      count: DEFAULT_LLM_LIST.length,
+    });
     assert
       .dom('[data-test-llm-select-item="google/gemini-pro-1.5"]')
       .hasText('google/gemini-pro-1.5');

Diff for: packages/runtime-common/matrix-constants.ts

@@ -20,4 +20,6 @@ export const DEFAULT_LLM_LIST = [
   'google/gemini-pro-1.5',
   'openai/gpt-4o',
   'openai/gpt-4o-mini',
+  'deepseek/deepseek-r1',
+  'deepseek/deepseek-r1-distill-llama-70b',
 ];