From 86c5a94b14db864abd3b5aa025b8a33baea69fbc Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 22 Nov 2024 17:48:04 +0700 Subject: [PATCH 01/10] tunning prompt --- helpers/env-variables.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/helpers/env-variables.ts b/helpers/env-variables.ts index 37fc2d537..470323a56 100644 --- a/helpers/env-variables.ts +++ b/helpers/env-variables.ts @@ -13,6 +13,12 @@ import { import { TSYSTEMS_LLMHUB_API_URL } from "./providers/llmhub"; +const DEFAULT_SYSTEM_PROMPT = + "You are a helpful assistant who helps users with their questions."; + +const DATA_SOURCES_PROMPT = + "You have access to a knowledge base, use query engine tool to retrieve the facts to answer the user question."; + export type EnvVar = { name?: string; description?: string; @@ -449,9 +455,6 @@ const getSystemPromptEnv = ( dataSources?: TemplateDataSource[], template?: TemplateType, ): EnvVar[] => { - const defaultSystemPrompt = - "You are a helpful assistant who helps users with their questions."; - const systemPromptEnv: EnvVar[] = []; // build tool system prompt by merging all tool system prompts // multiagent template doesn't need system prompt @@ -466,9 +469,12 @@ const getSystemPromptEnv = ( } }); - const systemPrompt = toolSystemPrompt - ? `\"${toolSystemPrompt}\"` - : defaultSystemPrompt; + const systemPrompt = + "'" + + DEFAULT_SYSTEM_PROMPT + + (dataSources?.length ? `\n${DATA_SOURCES_PROMPT}` : "") + + (toolSystemPrompt ? `\n${toolSystemPrompt}` : "") + + "'"; systemPromptEnv.push({ name: "SYSTEM_PROMPT", From 2e65ba36e07beeb6fd38105af280d80534af1189 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 22 Nov 2024 17:51:49 +0700 Subject: [PATCH 02/10] add error handling on streaming --- .../app/api/routers/vercel_response.py | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py index 1155f6ba7..2f358873d 100644 --- a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py +++ b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py @@ -53,17 +53,27 @@ async def content_generator( # Merge the chat response generator and the event generator combine = stream.merge(chat_response_generator, event_generator) is_stream_started = False - async with combine.stream() as streamer: - async for output in streamer: - if not is_stream_started: - is_stream_started = True - # Stream a blank message to start displaying the response in the UI - yield cls.convert_text("") - - yield output - - if await request.is_disconnected(): - break + try: + async with combine.stream() as streamer: + async for output in streamer: + if await request.is_disconnected(): + break + + if not is_stream_started: + is_stream_started = True + # Stream a blank message to start displaying the response in the UI + yield cls.convert_text("") + + yield output + except Exception: + logger.exception("Error in stream response") + error_message = ( + "An error occurred while processing your request. Please try again." + ) + yield cls.convert_text(error_message) + finally: + # Ensure event handler is marked as done even if connection breaks + event_handler.is_done = True @classmethod async def _event_generator(cls, event_handler: EventCallbackHandler): From f437b46992e13a8001ecec783a3882e942be38dd Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 25 Nov 2024 11:35:32 +0700 Subject: [PATCH 03/10] update tools prompt --- helpers/tools.ts | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/helpers/tools.ts b/helpers/tools.ts index 5d3ab929f..c3a34390c 100644 --- a/helpers/tools.ts +++ b/helpers/tools.ts @@ -71,8 +71,7 @@ export const supportedTools: Tool[] = [ { name: TOOL_SYSTEM_PROMPT_ENV_VAR, description: "System prompt for DuckDuckGo search tool.", - value: `You are a DuckDuckGo search agent. -You can use the duckduckgo search tool to get information from the web to answer user questions. + value: `You have access to the duckduckgo search tool. Use it to get information from the web to answer user questions. For better results, you can specify the region parameter to get results from a specific region but it's optional.`, }, ], @@ -88,13 +87,6 @@ For better results, you can specify the region parameter to get results from a s ], supportedFrameworks: ["fastapi", "express", "nextjs"], type: ToolType.LLAMAHUB, - envVars: [ - { - name: TOOL_SYSTEM_PROMPT_ENV_VAR, - description: "System prompt for wiki tool.", - value: `You are a Wikipedia agent. You help users to get information from Wikipedia.`, - }, - ], }, { display: "Weather", @@ -102,13 +94,6 @@ For better results, you can specify the region parameter to get results from a s dependencies: [], supportedFrameworks: ["fastapi", "express", "nextjs"], type: ToolType.LOCAL, - envVars: [ - { - name: TOOL_SYSTEM_PROMPT_ENV_VAR, - description: "System prompt for weather tool.", - value: `You are a weather forecast agent. You help users to get the weather forecast for a given location.`, - }, - ], }, { display: "Document generator", @@ -211,14 +196,6 @@ For better results, you can specify the region parameter to get results from a s }, supportedFrameworks: ["fastapi", "express", "nextjs"], type: ToolType.LOCAL, - envVars: [ - { - name: TOOL_SYSTEM_PROMPT_ENV_VAR, - description: "System prompt for openapi action tool.", - value: - "You are an OpenAPI action agent. You help users to make requests to the provided OpenAPI schema.", - }, - ], }, { display: "Image Generator", @@ -231,11 +208,6 @@ For better results, you can specify the region parameter to get results from a s description: "STABILITY_API_KEY key is required to run image generator. Get it here: https://platform.stability.ai/account/keys", }, - { - name: TOOL_SYSTEM_PROMPT_ENV_VAR, - description: "System prompt for image generator tool.", - value: `You are an image generator agent. You help users to generate images using the Stability API.`, - }, ], }, { From 1bac1be0651d7a0e456d4655402af5dee247beef Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 25 Nov 2024 13:06:31 +0700 Subject: [PATCH 04/10] update error handling on streaming --- .../python/app/api/routers/vercel_response.py | 8 ++++++++ .../fastapi/app/api/routers/vercel_response.py | 11 ++++++++--- .../streaming/nextjs/app/components/chat-section.tsx | 8 +++++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/templates/components/multiagent/python/app/api/routers/vercel_response.py b/templates/components/multiagent/python/app/api/routers/vercel_response.py index 429855324..599929658 100644 --- a/templates/components/multiagent/python/app/api/routers/vercel_response.py +++ b/templates/components/multiagent/python/app/api/routers/vercel_response.py @@ -47,6 +47,9 @@ async def content_generator(self, event_handler, events): logger.error( f"Unexpected error in content_generator: {str(e)}", exc_info=True ) + yield self.convert_error( + "An unexpected error occurred while processing your request, preventing the creation of a final answer. Please try again." + ) finally: logger.info("The stream has been stopped!") @@ -107,6 +110,11 @@ def convert_data(cls, data: dict): data_str = json.dumps(data) return f"{cls.DATA_PREFIX}[{data_str}]\n" + @classmethod + def convert_error(cls, error: str): + error_str = json.dumps(error) + return f"{cls.ERROR_PREFIX}{error_str}\n" + @staticmethod async def _generate_next_questions(chat_history: List[Message], response: str): questions = await NextQuestionSuggestion.suggest_next_questions( diff --git a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py index 2f358873d..0d41d893e 100644 --- a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py +++ b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py @@ -22,6 +22,7 @@ class VercelStreamResponse(StreamingResponse): TEXT_PREFIX = "0:" DATA_PREFIX = "8:" + ERROR_PREFIX = "3:" def __init__( self, @@ -67,10 +68,9 @@ async def content_generator( yield output except Exception: logger.exception("Error in stream response") - error_message = ( - "An error occurred while processing your request. Please try again." + yield cls.convert_error( + "An unexpected error occurred while processing your request, preventing the creation of a final answer. Please try again." ) - yield cls.convert_text(error_message) finally: # Ensure event handler is marked as done even if connection breaks event_handler.is_done = True @@ -141,6 +141,11 @@ def convert_data(cls, data: dict): data_str = json.dumps(data) return f"{cls.DATA_PREFIX}[{data_str}]\n" + @classmethod + def convert_error(cls, error: str): + error_str = json.dumps(error) + return f"{cls.ERROR_PREFIX}{error_str}\n" + @staticmethod def _process_response_nodes( source_nodes: List[NodeWithScore], diff --git a/templates/types/streaming/nextjs/app/components/chat-section.tsx b/templates/types/streaming/nextjs/app/components/chat-section.tsx index 46b005427..4a0fe9deb 100644 --- a/templates/types/streaming/nextjs/app/components/chat-section.tsx +++ b/templates/types/streaming/nextjs/app/components/chat-section.tsx @@ -15,7 +15,13 @@ export default function ChatSection() { api: `${backend}/api/chat`, onError: (error: unknown) => { if (!(error instanceof Error)) throw error; - alert(JSON.parse(error.message).detail); + let errorMessage: string; + try { + errorMessage = JSON.parse(error.message).detail; + } catch (e) { + errorMessage = error.message; + } + alert(errorMessage); }, }); return ( From 126e1d9285143ed0a95db0dde2a741fd59305d97 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 25 Nov 2024 13:35:06 +0700 Subject: [PATCH 05/10] update e2b file upload code to fix e2e --- .../components/engines/typescript/agent/tools/interpreter.ts | 4 +++- templates/types/streaming/nextjs/app/api/sandbox/route.ts | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/templates/components/engines/typescript/agent/tools/interpreter.ts b/templates/components/engines/typescript/agent/tools/interpreter.ts index 44cc7cbc4..37b645882 100644 --- a/templates/components/engines/typescript/agent/tools/interpreter.ts +++ b/templates/components/engines/typescript/agent/tools/interpreter.ts @@ -116,7 +116,9 @@ export class InterpreterTool implements BaseTool { const fileName = path.basename(filePath); const localFilePath = path.join(this.uploadedFilesDir, fileName); const content = fs.readFileSync(localFilePath); - await this.codeInterpreter?.files.write(filePath, content); + + const arrayBuffer = new Uint8Array(content).buffer; + await this.codeInterpreter?.files.write(filePath, arrayBuffer); } } catch (error) { console.error("Got error when uploading files to sandbox", error); diff --git a/templates/types/streaming/nextjs/app/api/sandbox/route.ts b/templates/types/streaming/nextjs/app/api/sandbox/route.ts index 6bbd15177..f524d5507 100644 --- a/templates/types/streaming/nextjs/app/api/sandbox/route.ts +++ b/templates/types/streaming/nextjs/app/api/sandbox/route.ts @@ -92,7 +92,8 @@ export async function POST(req: Request) { const localFilePath = path.join("output", "uploaded", fileName); const fileContent = await fs.readFile(localFilePath); - await sbx.files.write(sandboxFilePath, fileContent); + const arrayBuffer = new Uint8Array(fileContent).buffer; + await sbx.files.write(sandboxFilePath, arrayBuffer); console.log(`Copied file to ${sandboxFilePath} in ${sbx.sandboxID}`); }); } From d862441c60735dc4fcba4f2227f06b92c082cff8 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 25 Nov 2024 13:38:17 +0700 Subject: [PATCH 06/10] fix missing variable --- .../multiagent/python/app/api/routers/vercel_response.py | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/components/multiagent/python/app/api/routers/vercel_response.py b/templates/components/multiagent/python/app/api/routers/vercel_response.py index 599929658..3547d459c 100644 --- a/templates/components/multiagent/python/app/api/routers/vercel_response.py +++ b/templates/components/multiagent/python/app/api/routers/vercel_response.py @@ -19,6 +19,7 @@ class VercelStreamResponse(StreamingResponse): TEXT_PREFIX = "0:" DATA_PREFIX = "8:" + ERROR_PREFIX = "3:" def __init__(self, request: Request, chat_data: ChatData, *args, **kwargs): self.request = request From 669d492cdb11be3df00df3c27d25b4eed8cd94b3 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 25 Nov 2024 13:49:31 +0700 Subject: [PATCH 07/10] add cancel run for workflow --- .../multiagent/python/app/api/routers/vercel_response.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/components/multiagent/python/app/api/routers/vercel_response.py b/templates/components/multiagent/python/app/api/routers/vercel_response.py index 3547d459c..0fd1d0172 100644 --- a/templates/components/multiagent/python/app/api/routers/vercel_response.py +++ b/templates/components/multiagent/python/app/api/routers/vercel_response.py @@ -42,8 +42,7 @@ async def content_generator(self, event_handler, events): yield output except asyncio.CancelledError: - logger.info("Stopping workflow") - await event_handler.cancel_run() + logger.warning("Workflow has been cancelled!") except Exception as e: logger.error( f"Unexpected error in content_generator: {str(e)}", exc_info=True @@ -52,6 +51,7 @@ async def content_generator(self, event_handler, events): "An unexpected error occurred while processing your request, preventing the creation of a final answer. Please try again." ) finally: + await event_handler.cancel_run() logger.info("The stream has been stopped!") def _create_stream( From e4233590dd59779fe9c54c39b4d4bc3418d1fec7 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 25 Nov 2024 14:25:19 +0700 Subject: [PATCH 08/10] improve system prompt --- helpers/env-variables.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/env-variables.ts b/helpers/env-variables.ts index 470323a56..5e430b2b1 100644 --- a/helpers/env-variables.ts +++ b/helpers/env-variables.ts @@ -17,7 +17,7 @@ const DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant who helps users with their questions."; const DATA_SOURCES_PROMPT = - "You have access to a knowledge base, use query engine tool to retrieve the facts to answer the user question."; + "You have access to a knowledge base including the facts that you should start with to find the answer for the user question. Use the query engine tool to retrieve the facts from the knowledge base."; export type EnvVar = { name?: string; From 97b07d57f54387bb6a7bbea828f83e5a499e073f Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 25 Nov 2024 17:22:48 +0700 Subject: [PATCH 09/10] add missing --- .changeset/moody-mangos-punch.md | 5 +++++ questions/simple.ts | 2 +- templates/types/streaming/express/README-template.md | 2 +- templates/types/streaming/fastapi/README-template.md | 2 +- templates/types/streaming/nextjs/README-template.md | 2 +- 5 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 .changeset/moody-mangos-punch.md diff --git a/.changeset/moody-mangos-punch.md b/.changeset/moody-mangos-punch.md new file mode 100644 index 000000000..3a132863d --- /dev/null +++ b/.changeset/moody-mangos-punch.md @@ -0,0 +1,5 @@ +--- +"create-llama": patch +--- + +Improve system prompt and change agentic default tool to weather diff --git a/questions/simple.ts b/questions/simple.ts index 8571021fc..e7892acdd 100644 --- a/questions/simple.ts +++ b/questions/simple.ts @@ -131,7 +131,7 @@ const convertAnswers = async ( > = { rag: { template: "streaming", - tools: getTools(["wikipedia.WikipediaToolSpec"]), + tools: getTools(["weather"]), frontend: true, dataSources: [EXAMPLE_FILE], }, diff --git a/templates/types/streaming/express/README-template.md b/templates/types/streaming/express/README-template.md index 8ec046643..4b9ece10e 100644 --- a/templates/types/streaming/express/README-template.md +++ b/templates/types/streaming/express/README-template.md @@ -8,7 +8,7 @@ First, install the dependencies: npm install ``` -Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step): +Second, generate the embeddings of the documents in the `./data` directory: ``` npm run generate diff --git a/templates/types/streaming/fastapi/README-template.md b/templates/types/streaming/fastapi/README-template.md index 00a522e7e..ff9227077 100644 --- a/templates/types/streaming/fastapi/README-template.md +++ b/templates/types/streaming/fastapi/README-template.md @@ -15,7 +15,7 @@ Then check the parameters that have been pre-configured in the `.env` file in th If you are using any tools or data sources, you can update their config files in the `config` folder. -Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step): +Second, generate the embeddings of the documents in the `./data` directory: ``` poetry run generate diff --git a/templates/types/streaming/nextjs/README-template.md b/templates/types/streaming/nextjs/README-template.md index d2eb1eb85..78a392813 100644 --- a/templates/types/streaming/nextjs/README-template.md +++ b/templates/types/streaming/nextjs/README-template.md @@ -8,7 +8,7 @@ First, install the dependencies: npm install ``` -Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step): +Second, generate the embeddings of the documents in the `./data` directory: ``` npm run generate From e0f224def611635d972fef24153f5de768149fd4 Mon Sep 17 00:00:00 2001 From: Marcus Schiesser Date: Tue, 26 Nov 2024 10:53:52 +0700 Subject: [PATCH 10/10] Update .changeset/moody-mangos-punch.md --- .changeset/moody-mangos-punch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/moody-mangos-punch.md b/.changeset/moody-mangos-punch.md index 3a132863d..b54e1527a 100644 --- a/.changeset/moody-mangos-punch.md +++ b/.changeset/moody-mangos-punch.md @@ -2,4 +2,4 @@ "create-llama": patch --- -Improve system prompt and change agentic default tool to weather +Show streaming errors in Python, optimize system prompts for tool usage and set the weather tool as default for the Agentic RAG use case