feat: implement artifact tool in TS

.changeset/modern-cars-travel.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+feat: implement artifact tool in TS

helpers/tools.ts

@@ -139,6 +139,27 @@ For better results, you can specify the region parameter to get results from a s
       },
     ],
   },
+  {
+    display: "Artifact",
+    name: "artifact",
+    dependencies: [],
+    supportedFrameworks: ["express", "nextjs"],
+    type: ToolType.LOCAL,
+    envVars: [
+      {
+        name: "E2B_API_KEY",
+        description:
+          "E2B_API_KEY key is required to run code interpreter tool. Get it here: https://e2b.dev/docs/getting-started/api-key",
+      },
+      {
+        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
+        description: "System prompt for code interpreter tool.",
+        value: `You are a code executor that can run code in a secure environment.
+You can run code in a sandbox and return the sandbox url to the user for them to view the result.
+Do not show the code in chat, just describe the detail information of application: step by step to build the application, title, description, dependencies and sandbox url`,
+      },
+    ],
+  },
   {
     display: "OpenAPI action",
     name: "openapi_action.OpenAPIActionToolSpec",

templates/components/engines/typescript/agent/tools/artifact.ts

@@ -0,0 +1,158 @@
+import { CodeInterpreter, Sandbox } from "@e2b/code-interpreter";
+import type { JSONSchemaType } from "ajv";
+import {
+  BaseTool,
+  ChatMessage,
+  JSONValue,
+  Settings,
+  ToolMetadata,
+} from "llamaindex";
+
+const CODE_GENERATION_PROMPT = `You are a skilled software engineer. You do not make mistakes. Generate an artifact. You can install additional dependencies. You can use one of the following templates:\n
+
+1. code-interpreter-multilang: "Runs code as a Jupyter notebook cell. Strong data analysis angle. Can use complex visualisation to explain results.". File: script.py. Dependencies installed: python, jupyter, numpy, pandas, matplotlib, seaborn, plotly. Port: none.
+
+2. nextjs-developer: "A Next.js 13+ app that reloads automatically. Using the pages router.". File: pages/index.tsx. Dependencies installed: nextjs@14.2.5, typescript, @types/node, @types/react, @types/react-dom, postcss, tailwindcss, shadcn. Port: 3000.
+
+3. vue-developer: "A Vue.js 3+ app that reloads automatically. Only when asked specifically for a Vue app.". File: app.vue. Dependencies installed: vue@latest, nuxt@3.13.0, tailwindcss. Port: 3000.
+
+4. streamlit-developer: "A streamlit app that reloads automatically.". File: app.py. Dependencies installed: streamlit, pandas, numpy, matplotlib, request, seaborn, plotly. Port: 8501.
+
+5. gradio-developer: "A gradio app. Gradio Blocks/Interface should be called demo.". File: app.py. Dependencies installed: gradio, pandas, numpy, matplotlib, request, seaborn, plotly. Port: 7860.
+
+Provide detail information about the artifact you're about to generate in the following JSON format with the following keys:
+
+commentary: Describe what you're about to do and the steps you want to take for generating the artifact in great detail.
+template: Name of the template used to generate the artifact.
+title: Short title of the artifact. Max 3 words.
+description: Short description of the artifact. Max 1 sentence.
+additional_dependencies: Additional dependencies required by the artifact. Do not include dependencies that are already included in the template.
+has_additional_dependencies: Detect if additional dependencies that are not included in the template are required by the artifact.
+install_dependencies_command: Command to install additional dependencies required by the artifact.
+port: Port number used by the resulted artifact. Null when no ports are exposed.
+file_path: Relative path to the file, including the file name.
+code: Code generated by the artifact. Only runnable code is allowed.
+
+Make sure to use the correct syntax for the programming language you're using.
+`;
+
+// detail information to execute code in sandbox
+export type Artifact = {
+  commentary: string;
+  template: string;
+  title: string;
+  description: string;
+  additional_dependencies: string[];
+  has_additional_dependencies: boolean;
+  install_dependencies_command: string;
+  port: number | null;
+  file_path: string;
+  code: string;
+};
+
+export type ArtifactParameter = {
+  requirement: string;
+};
+
+export type ArtifactToolParams = {
+  metadata?: ToolMetadata<JSONSchemaType<ArtifactParameter>>;
+  apiKey?: string;
+  timeout?: number;
+};
+
+const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<ArtifactParameter>> = {
+  name: "artifact",
+  description: `Generate an artifact based on the input then execute it and return sandbox url.`,
+  parameters: {
+    type: "object",
+    properties: {
+      requirement: {
+        type: "string",
+        description: "The description of the application you want to build.",
+      },
+    },
+    required: ["requirement"],
+  },
+};
+
+export class ArtifactTool implements BaseTool<ArtifactParameter> {
+  metadata: ToolMetadata<JSONSchemaType<ArtifactParameter>>;
+  apiKey: string;
+  timeout: number;
+
+  constructor(params?: ArtifactToolParams) {
+    this.metadata = params?.metadata || DEFAULT_META_DATA;
+    this.timeout = params?.timeout || 10 * 60 * 1000; // 10 minutes in ms
+    this.apiKey = params?.apiKey || process.env.E2B_API_KEY || "";
+
+    if (!this.apiKey) {
+      throw new Error("E2B_API_KEY is not set");
+    }
+  }
+
+  async call(input: ArtifactParameter) {
+    try {
+      const artifact = await this.generateArtifact(input.requirement);
+      const url = await this.executeArtifact(artifact);
+      return { artifact, url } as JSONValue;
+    } catch (error) {
+      return {};
+    }
+  }
+
+  // Generate artifact (code, environment, dependencies, etc.)
+  async generateArtifact(query: string): Promise<Artifact> {
+    const messages: ChatMessage[] = [
+      { role: "system", content: CODE_GENERATION_PROMPT },
+      { role: "user", content: query },
+    ];
+    try {
+      const response = await Settings.llm.chat({ messages });
+      const artifact = JSON.parse(
+        response.message.content.toString(),
+      ) as Artifact;
+      return artifact;
+    } catch (error) {
+      console.log("Failed to generate artifact", error);
+      throw error;
+    }
+  }
+
+  // Execute artifact in sandbox and return the URL
+  async executeArtifact(artifact: Artifact): Promise<string> {
+    try {
+      // Create sandbox
+      const sbx = await Sandbox.create(artifact.template, {
+        metadata: { template: artifact.template, userID: "default" },
+        timeoutMs: this.timeout,
+        apiKey: this.apiKey,
+      });
+      console.log("Created sandbox", sbx.sandboxID);
+
+      // Install packages
+      if (artifact.has_additional_dependencies) {
+        if (sbx instanceof CodeInterpreter) {
+          await sbx.notebook.execCell(artifact.install_dependencies_command);
+          console.log(
+            `Installed dependencies: ${artifact.additional_dependencies.join(", ")} in code interpreter ${sbx.sandboxID}`,
+          );
+        } else if (sbx instanceof Sandbox) {
+          await sbx.commands.run(artifact.install_dependencies_command);
+          console.log(
+            `Installed dependencies: ${artifact.additional_dependencies.join(", ")} in sandbox ${sbx.sandboxID}`,
+          );
+        }
+      }
+
+      // Copy code to fs
+      await sbx.files.write(artifact.file_path, artifact.code);
+      console.log(`Copied file to ${artifact.file_path} in ${sbx.sandboxID}`);
+
+      // Execute code or return a URL to the running sandbox
+      return `https://${sbx?.getHost(artifact.port || 80)}`;
+    } catch (error) {
+      console.log("Failed to execute artifact", error);
+      throw error;
+    }
+  }
+}

templates/components/engines/typescript/agent/tools/index.ts

@@ -1,5 +1,6 @@
 import { BaseToolWithCall } from "llamaindex";
 import { ToolsFactory } from "llamaindex/tools/ToolsFactory";
+import { ArtifactTool, ArtifactToolParams } from "./artifact";
 import { DuckDuckGoSearchTool, DuckDuckGoToolParams } from "./duckduckgo";
 import { ImgGeneratorTool, ImgGeneratorToolParams } from "./img-gen";
 import { InterpreterTool, InterpreterToolParams } from "./interpreter";
@@ -43,6 +44,9 @@ const toolFactory: Record<string, ToolCreator> = {
   img_gen: async (config: unknown) => {
     return [new ImgGeneratorTool(config as ImgGeneratorToolParams)];
   },
+  artifact: async (config: unknown) => {
+    return [new ArtifactTool(config as ArtifactToolParams)];
+  },
 };
 
 async function createLocalTools(

templates/types/streaming/express/package.json

@@ -23,7 +23,7 @@
     "llamaindex": "0.6.2",
     "pdf2json": "3.0.5",
     "ajv": "^8.12.0",
-    "@e2b/code-interpreter": "^0.0.5",
+    "@e2b/code-interpreter": "0.0.9-beta.3",
     "got": "^14.4.1",
     "@apidevtools/swagger-parser": "^10.1.0",
     "formdata-node": "^6.0.3"

templates/types/streaming/nextjs/app/components/ui/chat/chat-message/chat-tools.tsx

@@ -1,4 +1,5 @@
 import { ToolData } from "../index";
+import { Artifact } from "../widgets/Artifact";
 import { WeatherCard, WeatherData } from "../widgets/WeatherCard";
 
 // TODO: If needed, add displaying more tool outputs here
@@ -20,6 +21,8 @@ export default function ChatTools({ data }: { data: ToolData }) {
     case "get_weather_information":
       const weatherData = toolOutput.output as unknown as WeatherData;
       return <WeatherCard data={weatherData} />;
+    case "artifact":
+      return <Artifact data={toolOutput.output} />;
     default:
       return null;
   }

templates/types/streaming/nextjs/app/components/ui/chat/widgets/Artifact.tsx

@@ -0,0 +1,86 @@
+import { JSONValue } from "ai";
+import { Rocket } from "lucide-react";
+import { Button } from "../../button";
+import {
+  Drawer,
+  DrawerClose,
+  DrawerContent,
+  DrawerDescription,
+  DrawerHeader,
+  DrawerTitle,
+  DrawerTrigger,
+} from "../../drawer";
+import { Tabs, TabsContent, TabsList, TabsTrigger } from "../../tabs";
+import Markdown from "../chat-message/markdown";
+
+type Artifact = {
+  commentary: string;
+  template: string;
+  title: string;
+  description: string;
+  additional_dependencies: string[];
+  has_additional_dependencies: boolean;
+  install_dependencies_command: string;
+  port: number | null;
+  file_path: string;
+  code: string;
+};
+
+type ArtifactProps = {
+  artifact?: Artifact;
+  url?: string;
+};
+
+export function Artifact({ data }: { data: JSONValue }) {
+  const { artifact, url } = data as ArtifactProps;
+  if (!artifact || !url) return null;
+
+  const fileExtension = artifact.file_path.split(".").pop();
+  const markdownCode = `\`\`\`${fileExtension}\n${artifact.code}\n\`\`\``;
+
+  return (
+    <Drawer direction="left">
+      <DrawerTrigger asChild>
+        <Button className="w-max">
+          View Demo <Rocket className="ml-2 h-4 w-4" />
+        </Button>
+      </DrawerTrigger>
+      <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] ">
+        <DrawerHeader className="flex justify-between">
+          <div className="space-y-2">
+            <DrawerTitle>Artifact Preview</DrawerTitle>
+            <DrawerDescription>
+              Open in your brower:{" "}
+              <a href={url} target="_blank" className="text-blue-500">
+                {url}
+              </a>
+            </DrawerDescription>
+          </div>
+          <DrawerClose asChild>
+            <Button variant="outline">Close</Button>
+          </DrawerClose>
+        </DrawerHeader>
+        <Tabs defaultValue="code" className="h-full p-4 overflow-auto">
+          <TabsList className="grid w-full grid-cols-2 max-w-[400px]">
+            <TabsTrigger value="code">Code</TabsTrigger>
+            <TabsTrigger value="preview">Preview</TabsTrigger>
+          </TabsList>
+          <TabsContent value="code" className="h-full">
+            <div className="m-4 overflow-auto">
+              <Markdown content={markdownCode} />
+            </div>
+          </TabsContent>
+          <TabsContent value="preview" className="h-full">
+            <iframe
+              key={url}
+              className="h-full w-full"
+              sandbox="allow-forms allow-scripts allow-same-origin"
+              loading="lazy"
+              src={url}
+            />
+          </TabsContent>
+        </Tabs>
+      </DrawerContent>
+    </Drawer>
+  );
+}

templates/types/streaming/nextjs/app/components/ui/tabs.tsx

@@ -0,0 +1,54 @@
+"use client";
+
+import * as TabsPrimitive from "@radix-ui/react-tabs";
+import * as React from "react";
+import { cn } from "./lib/utils";
+
+const Tabs = TabsPrimitive.Root;
+
+const TabsList = React.forwardRef<
+  React.ElementRef<typeof TabsPrimitive.List>,
+  React.ComponentPropsWithoutRef<typeof TabsPrimitive.List>
+>(({ className, ...props }, ref) => (
+  <TabsPrimitive.List
+    ref={ref}
+    className={cn(
+      "inline-flex h-9 items-center justify-center rounded-lg bg-muted p-1 text-muted-foreground",
+      className,
+    )}
+    {...props}
+  />
+));
+TabsList.displayName = TabsPrimitive.List.displayName;
+
+const TabsTrigger = React.forwardRef<
+  React.ElementRef<typeof TabsPrimitive.Trigger>,
+  React.ComponentPropsWithoutRef<typeof TabsPrimitive.Trigger>
+>(({ className, ...props }, ref) => (
+  <TabsPrimitive.Trigger
+    ref={ref}
+    className={cn(
+      "inline-flex items-center justify-center whitespace-nowrap rounded-md px-3 py-1 text-sm font-medium ring-offset-background transition-all focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 data-[state=active]:bg-background data-[state=active]:text-foreground data-[state=active]:shadow",
+      className,
+    )}
+    {...props}
+  />
+));
+TabsTrigger.displayName = TabsPrimitive.Trigger.displayName;
+
+const TabsContent = React.forwardRef<
+  React.ElementRef<typeof TabsPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof TabsPrimitive.Content>
+>(({ className, ...props }, ref) => (
+  <TabsPrimitive.Content
+    ref={ref}
+    className={cn(
+      "ring-offset-background focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2",
+      className,
+    )}
+    {...props}
+  />
+));
+TabsContent.displayName = TabsPrimitive.Content.displayName;
+
+export { Tabs, TabsContent, TabsList, TabsTrigger };

templates/types/streaming/nextjs/package.json

@@ -11,12 +11,13 @@
   },
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.1.0",
-    "@e2b/code-interpreter": "^0.0.5",
+    "@e2b/code-interpreter": "0.0.9-beta.3",
     "@llamaindex/pdf-viewer": "^1.1.3",
     "@radix-ui/react-collapsible": "^1.0.3",
     "@radix-ui/react-hover-card": "^1.0.7",
     "@radix-ui/react-select": "^2.1.1",
     "@radix-ui/react-slot": "^1.0.2",
+    "@radix-ui/react-tabs": "^1.1.0",
     "ai": "3.3.42",
     "ajv": "^8.12.0",
     "class-variance-authority": "^0.7.0",