feat: implement artifact tool in TS

.changeset/modern-cars-travel.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+feat: implement artifact tool in TS

e2e/shared/multiagent_template.spec.ts

@@ -66,7 +66,7 @@ test.describe(`Test multiagent template ${templateFramework} ${dataSource} ${tem
     page,
   }) => {
     await page.goto(`http://localhost:${port}`);
-    await page.fill("form input", userMessage);
+    await page.fill("form textarea", userMessage);
 
     const responsePromise = page.waitForResponse((res) =>
       res.url().includes("/api/chat"),

e2e/shared/streaming_template.spec.ts

@@ -72,7 +72,7 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
   }) => {
     test.skip(templatePostInstallAction !== "runApp");
     await page.goto(`http://localhost:${port}`);
-    await page.fill("form input", userMessage);
+    await page.fill("form textarea", userMessage);
     const [response] = await Promise.all([
       page.waitForResponse(
         (res) => {

helpers/tools.ts

@@ -139,6 +139,27 @@ For better results, you can specify the region parameter to get results from a s
       },
     ],
   },
+  {
+    display: "Artifact",
+    name: "artifact",
+    dependencies: [],
+    supportedFrameworks: ["express", "nextjs"],
+    type: ToolType.LOCAL,
+    envVars: [
+      {
+        name: "E2B_API_KEY",
+        description:
+          "E2B_API_KEY key is required to run code interpreter tool. Get it here: https://e2b.dev/docs/getting-started/api-key",
+      },
+      {
+        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
+        description: "System prompt for code interpreter tool.",
+        value: `You are a code executor that can run code in a secure environment.
+You can run code in a sandbox and return the sandbox url to the user for them to view the result.
+Do not show the code or sandbox url in chat, just describe the steps to build the application.`,
+      },
+    ],
+  },
   {
     display: "OpenAPI action",
     name: "openapi_action.OpenAPIActionToolSpec",

templates/components/engines/typescript/agent/tools/code-generator.ts

@@ -0,0 +1,129 @@
+import type { JSONSchemaType } from "ajv";
+import {
+  BaseTool,
+  ChatMessage,
+  JSONValue,
+  Settings,
+  ToolMetadata,
+} from "llamaindex";
+
+// prompt based on https://github.com/e2b-dev/ai-artifacts
+const CODE_GENERATION_PROMPT = `You are a skilled software engineer. You do not make mistakes. Generate an artifact. You can install additional dependencies. You can use one of the following templates:\n
+
+1. code-interpreter-multilang: "Runs code as a Jupyter notebook cell. Strong data analysis angle. Can use complex visualisation to explain results.". File: script.py. Dependencies installed: python, jupyter, numpy, pandas, matplotlib, seaborn, plotly. Port: none.
+
+2. nextjs-developer: "A Next.js 13+ app that reloads automatically. Using the pages router.". File: pages/index.tsx. Dependencies installed: nextjs@14.2.5, typescript, @types/node, @types/react, @types/react-dom, postcss, tailwindcss, shadcn. Port: 3000.
+
+3. vue-developer: "A Vue.js 3+ app that reloads automatically. Only when asked specifically for a Vue app.". File: app.vue. Dependencies installed: vue@latest, nuxt@3.13.0, tailwindcss. Port: 3000.
+
+4. streamlit-developer: "A streamlit app that reloads automatically.". File: app.py. Dependencies installed: streamlit, pandas, numpy, matplotlib, request, seaborn, plotly. Port: 8501.
+
+5. gradio-developer: "A gradio app. Gradio Blocks/Interface should be called demo.". File: app.py. Dependencies installed: gradio, pandas, numpy, matplotlib, request, seaborn, plotly. Port: 7860.
+
+Provide detail information about the artifact you're about to generate in the following JSON format with the following keys:
+
+commentary: Describe what you're about to do and the steps you want to take for generating the artifact in great detail.
+template: Name of the template used to generate the artifact.
+title: Short title of the artifact. Max 3 words.
+description: Short description of the artifact. Max 1 sentence.
+additional_dependencies: Additional dependencies required by the artifact. Do not include dependencies that are already included in the template.
+has_additional_dependencies: Detect if additional dependencies that are not included in the template are required by the artifact.
+install_dependencies_command: Command to install additional dependencies required by the artifact.
+port: Port number used by the resulted artifact. Null when no ports are exposed.
+file_path: Relative path to the file, including the file name.
+code: Code generated by the artifact. Only runnable code is allowed.
+
+Make sure to use the correct syntax for the programming language you're using.
+`;
+
+// detail information to execute code
+export type CodeArtifact = {
+  commentary: string;
+  template: string;
+  title: string;
+  description: string;
+  additional_dependencies: string[];
+  has_additional_dependencies: boolean;
+  install_dependencies_command: string;
+  port: number | null;
+  file_path: string;
+  code: string;
+};
+
+export type CodeGeneratorParameter = {
+  requirement: string;
+  oldCode?: string;
+};
+
+export type CodeGeneratorToolParams = {
+  metadata?: ToolMetadata<JSONSchemaType<CodeGeneratorParameter>>;
+};
+
+const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<CodeGeneratorParameter>> =
+  {
+    name: "artifact",
+    description: `Generate a code artifact based on the input.`,
+    parameters: {
+      type: "object",
+      properties: {
+        requirement: {
+          type: "string",
+          description: "The description of the application you want to build.",
+        },
+        oldCode: {
+          type: "string",
+          description: "The existing code to be modified",
+          nullable: true,
+        },
+      },
+      required: ["requirement"],
+    },
+  };
+
+export class CodeGeneratorTool implements BaseTool<CodeGeneratorParameter> {
+  metadata: ToolMetadata<JSONSchemaType<CodeGeneratorParameter>>;
+
+  constructor(params?: CodeGeneratorToolParams) {
+    this.metadata = params?.metadata || DEFAULT_META_DATA;
+  }
+
+  async call(input: CodeGeneratorParameter) {
+    try {
+      const artifact = await this.generateArtifact(
+        input.requirement,
+        input.oldCode,
+      );
+      return artifact as JSONValue;
+    } catch (error) {
+      return { isError: true };
+    }
+  }
+
+  // Generate artifact (code, environment, dependencies, etc.)
+  async generateArtifact(
+    query: string,
+    oldCode?: string,
+  ): Promise<CodeArtifact> {
+    const userMessage = `
+    ${query}
+    ${oldCode ? `The existing code is: \n\`\`\`${oldCode}\`\`\`` : ""}
+    `;
+    const messages: ChatMessage[] = [
+      { role: "system", content: CODE_GENERATION_PROMPT },
+      { role: "user", content: userMessage },
+    ];
+    try {
+      const response = await Settings.llm.chat({ messages });
+      const content = response.message.content.toString();
+      const jsonContent = content
+        .replace(/^```json\s*|\s*```$/g, "")
+        .replace(/^`+|`+$/g, "")
+        .trim();
+      const artifact = JSON.parse(jsonContent) as CodeArtifact;
+      return artifact;
+    } catch (error) {
+      console.log("Failed to generate artifact", error);
+      throw error;
+    }
+  }
+}

templates/components/engines/typescript/agent/tools/index.ts

@@ -1,5 +1,6 @@
 import { BaseToolWithCall } from "llamaindex";
 import { ToolsFactory } from "llamaindex/tools/ToolsFactory";
+import { CodeGeneratorTool, CodeGeneratorToolParams } from "./code-generator";
 import { DuckDuckGoSearchTool, DuckDuckGoToolParams } from "./duckduckgo";
 import { ImgGeneratorTool, ImgGeneratorToolParams } from "./img-gen";
 import { InterpreterTool, InterpreterToolParams } from "./interpreter";
@@ -43,6 +44,9 @@ const toolFactory: Record<string, ToolCreator> = {
   img_gen: async (config: unknown) => {
     return [new ImgGeneratorTool(config as ImgGeneratorToolParams)];
   },
+  artifact: async (config: unknown) => {
+    return [new CodeGeneratorTool(config as CodeGeneratorToolParams)];
+  },
 };
 
 async function createLocalTools(

templates/components/llamaindex/typescript/streaming/annotations.ts

@@ -48,21 +48,54 @@ export function retrieveDocumentIds(annotations?: JSONValue[]): string[] {
   return ids;
 }
 
+// get latest artifact from annotations to append to the user message
+export function retrieveLatestArtifact(
+  allAnnotations?: JSONValue[],
+): string | undefined {
+  if (!allAnnotations) return undefined;
+
+  for (let index = allAnnotations.length - 1; index >= 0; index--) {
+    const annotation = allAnnotations[index];
+    const { type, data } = getValidAnnotation(annotation);
+    if (
+      type === "tools" &&
+      "toolCall" in data &&
+      "toolOutput" in data &&
+      typeof data.toolCall === "object" &&
+      typeof data.toolOutput === "object" &&
+      data.toolCall !== null &&
+      data.toolOutput !== null &&
+      "name" in data.toolCall &&
+      data.toolCall.name === "artifact"
+    ) {
+      const toolOutput = data.toolOutput as { output?: { code?: string } };
+      if (toolOutput.output?.code) {
+        return `The existing code is: \n\`\`\`${toolOutput.output.code}\`\`\``;
+      }
+    }
+  }
+  return undefined;
+}
+
 export function convertMessageContent(
   content: string,
   annotations?: JSONValue[],
+  latestArtifact?: string,
 ): MessageContent {
-  if (!annotations) return content;
   return [
     {
       type: "text",
       text: content,
     },
+    ...(latestArtifact
+      ? [{ type: "text" as const, text: latestArtifact }]
+      : []),
     ...convertAnnotations(annotations),
   ];
 }
 
-function convertAnnotations(annotations: JSONValue[]): MessageContentDetail[] {
+function convertAnnotations(annotations?: JSONValue[]): MessageContentDetail[] {
+  if (!annotations) return [];
   const content: MessageContentDetail[] = [];
   annotations.forEach((annotation: JSONValue) => {
     const { type, data } = getValidAnnotation(annotation);

templates/types/streaming/express/package.json

@@ -24,7 +24,7 @@
     "llamaindex": "0.6.2",
     "pdf2json": "3.0.5",
     "ajv": "^8.12.0",
-    "@e2b/code-interpreter": "^0.0.5",
+    "@e2b/code-interpreter": "0.0.9-beta.3",
     "got": "^14.4.1",
     "@apidevtools/swagger-parser": "^10.1.0",
     "formdata-node": "^6.0.3"

templates/types/streaming/nextjs/app/api/chat/route.ts

@@ -7,6 +7,7 @@ import { initSettings } from "./engine/settings";
 import {
   convertMessageContent,
   retrieveDocumentIds,
+  retrieveLatestArtifact,
 } from "./llamaindex/streaming/annotations";
 import {
   createCallbackManager,
@@ -59,12 +60,14 @@ export async function POST(request: NextRequest) {
       },
     );
     const ids = retrieveDocumentIds(allAnnotations);
+    const latestArtifact = retrieveLatestArtifact(allAnnotations);
     const chatEngine = await createChatEngine(ids, data);
 
     // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
     const userMessageContent = convertMessageContent(
       userMessage.content,
       annotations,
+      latestArtifact,
     );
 
     // Setup callbacks