diff --git a/chart/env/prod.yaml b/chart/env/prod.yaml index d9649f440f3..9b407506d37 100644 --- a/chart/env/prod.yaml +++ b/chart/env/prod.yaml @@ -50,7 +50,6 @@ envVars: [ { "name": "Qwen/Qwen2.5-72B-Instruct", - "tokenizer": "Qwen/Qwen2.5-72B-Instruct", "description": "The latest Qwen open model with improved role-playing, long text generation and structured data understanding.", "modelUrl": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct", "websiteUrl": "https://qwenlm.github.io/blog/qwen2.5/", @@ -62,6 +61,7 @@ envVars: "truncate": 28672, "max_new_tokens": 3072 }, + "tools": true, "promptExamples": [ { "title": "Write an email from bullet list", @@ -80,7 +80,6 @@ envVars: { "name": "meta-llama/Meta-Llama-3.1-70B-Instruct", "id": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "tokenizer": "nsarrazin/llama3.1-tokenizer", "description": "Ideal for everyday use. A fast and extremely capable model matching closed source models' capabilities.", "modelUrl": "https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct", "websiteUrl": "https://llama.meta.com/", @@ -110,7 +109,6 @@ envVars: }, { "name": "CohereForAI/c4ai-command-r-plus-08-2024", - "tokenizer": "nsarrazin/c4ai-command-r-v01-tokenizer", "description": "Cohere's largest language model, optimized for conversational interaction and tool use. Now with the 2024 update!", "modelUrl": "https://huggingface.co/CohereForAI/c4ai-command-r-plus-08-2024", "websiteUrl": "https://docs.cohere.com/docs/command-r-plus", @@ -350,10 +348,6 @@ envVars: }, { "name": "llhf/Meta-Llama-3.1-8B-Instruct", - "tokenizer": { - "tokenizerUrl": "https://huggingface.co/nsarrazin/llama3.1-tokenizer/resolve/main/tokenizer.json", - "tokenizerConfigUrl": "https://huggingface.co/nsarrazin/llama3.1-tokenizer/raw/main/tokenizer_config.json" - }, "parameters": { "temperature": 0.6, "stop": ["<|endoftext|>", "<|eot_id|>"] diff --git a/package-lock.json b/package-lock.json index 5daffe52b47..7319be5a7e9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,10 +10,10 @@ "dependencies": { "@aws-sdk/credential-providers": "^3.592.0", "@cliqz/adblocker-playwright": "^1.27.2", - "@gradio/client": "^1.1.1", + "@gradio/client": "^1.8.0", "@huggingface/hub": "^0.5.1", - "@huggingface/inference": "^2.7.0", - "@huggingface/transformers": "^3.0.0-alpha.6", + "@huggingface/inference": "^2.8.1", + "@huggingface/transformers": "^3.1.1", "@iconify-json/bi": "^1.1.21", "@playwright/browser-chromium": "^1.43.1", "@resvg/resvg-js": "^2.6.2", @@ -1929,9 +1929,10 @@ } }, "node_modules/@gradio/client": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/@gradio/client/-/client-1.5.2.tgz", - "integrity": "sha512-PJbt8n3ROxDXrR55D8oPYPIIkdmG5ksyljtgNM6Kr/FgTMTwGqiiYDdoyUIGwQSBu6g+jan2zbHoniDZePw3pg==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@gradio/client/-/client-1.8.0.tgz", + "integrity": "sha512-0Nxi+AuEL/ICdnZ6TSxRif1yS2A0HtJnggtWhN/8sqwFxx5UAGN29/vnJONcVfe3g9nctDJCvik3uDHkzQHFGw==", + "license": "ISC", "dependencies": { "@types/eventsource": "^1.1.15", "bufferutil": "^4.0.7", @@ -1959,37 +1960,41 @@ } }, "node_modules/@huggingface/inference": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.8.0.tgz", - "integrity": "sha512-Ti681P1qckcCAqgzmL53jBnluPuZGelmMIuXNjgAwC5+RIjF4S0SDQu6oy44ZTwekwNp2ETaZ2sXsOk+45aC4w==", + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.8.1.tgz", + "integrity": "sha512-EfsNtY9OR6JCNaUa5bZu2mrs48iqeTz0Gutwf+fU0Kypx33xFQB4DKMhp8u4Ee6qVbLbNWvTHuWwlppLQl4p4Q==", + "license": "MIT", "dependencies": { - "@huggingface/tasks": "^0.11.2" + "@huggingface/tasks": "^0.12.9" }, "engines": { "node": ">=18" } }, "node_modules/@huggingface/jinja": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.3.0.tgz", - "integrity": "sha512-GLJzso0M07ZncFkrJMIXVU4os6GFbPocD4g8fMQPMGJubf48FtGOsUORH2rtFdXPIPelz8SLBMn8ZRmOTwXm9Q==", + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.3.2.tgz", + "integrity": "sha512-F2FvuIc+w1blGsaqJI/OErRbWH6bVJDCBI8Rm5D86yZ2wlwrGERsfIaru7XUv9eYC3DMP3ixDRRtF0h6d8AZcQ==", + "license": "MIT", "engines": { "node": ">=18" } }, "node_modules/@huggingface/tasks": { - "version": "0.11.13", - "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.11.13.tgz", - "integrity": "sha512-TqFEyFtKYAYwDg9h4XQMzoSxN2NMpwSnerPBx7Y4RbM1nHLM+CTXAUHcDY+hydcA5CoDDaBGzkHw+mttY3AmFQ==" + "version": "0.12.30", + "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.12.30.tgz", + "integrity": "sha512-A1ITdxbEzx9L8wKR8pF7swyrTLxWNDFIGDLUWInxvks2ruQ8PLRBZe8r0EcjC3CDdtlj9jV1V4cgV35K/iy3GQ==", + "license": "MIT" }, "node_modules/@huggingface/transformers": { - "version": "3.0.0-alpha.14", - "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.0.0-alpha.14.tgz", - "integrity": "sha512-yRTKOP+xcZqHtlr7WiMWrV+E+XvqfiVLF4wZAlD1ozf0O5LPXKdkd8bfxFogd5d1kDxPlEdCLCJ3l8ZTNfqRyA==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.1.1.tgz", + "integrity": "sha512-/OpCiSKIowo5w5rJAOH3pgZKvpT6DOfDYw9br9Fp8w3qm4oyxc6dOhrxdRLVrmqLbE8rp5dKCePUW34ZBsinsg==", + "license": "Apache-2.0", "dependencies": { - "@huggingface/jinja": "^0.3.0", - "onnxruntime-node": "1.19.0", - "onnxruntime-web": "1.20.0-dev.20240827-1d059b8702", + "@huggingface/jinja": "^0.3.2", + "onnxruntime-node": "1.20.1", + "onnxruntime-web": "1.20.1", "sharp": "^0.33.5" } }, @@ -2639,6 +2644,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", "dependencies": { "minipass": "^7.0.4" }, @@ -2808,27 +2814,32 @@ "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" }, "node_modules/@protobufjs/base64": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" }, "node_modules/@protobufjs/codegen": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" }, "node_modules/@protobufjs/eventemitter": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" }, "node_modules/@protobufjs/fetch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", "dependencies": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" @@ -2837,27 +2848,32 @@ "node_modules/@protobufjs/float": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" }, "node_modules/@protobufjs/inquire": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" }, "node_modules/@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" }, "node_modules/@protobufjs/pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" }, "node_modules/@protobufjs/utf8": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" }, "node_modules/@remusao/guess-url-type": { "version": "1.3.0", @@ -5512,6 +5528,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", "engines": { "node": ">=18" } @@ -7103,7 +7120,8 @@ "node_modules/flatbuffers": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz", - "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==" + "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==", + "license": "SEE LICENSE IN LICENSE.txt" }, "node_modules/flatted": { "version": "3.3.1", @@ -7489,7 +7507,8 @@ "node_modules/guid-typescript": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", - "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==" + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" }, "node_modules/handlebars": { "version": "4.7.8", @@ -8749,7 +8768,8 @@ "node_modules/long": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", - "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==", + "license": "Apache-2.0" }, "node_modules/loupe": { "version": "3.1.2", @@ -8988,6 +9008,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", + "license": "MIT", "dependencies": { "minipass": "^7.0.4", "rimraf": "^5.0.5" @@ -9000,6 +9021,7 @@ "version": "5.0.10", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", "integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==", + "license": "ISC", "dependencies": { "glob": "^10.3.7" }, @@ -9677,43 +9699,41 @@ } }, "node_modules/onnxruntime-common": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.19.0.tgz", - "integrity": "sha512-Oo16UIJ/xLOtZDVGcL4bL8EP8MiNFztyBmR3pB14D+cl/UCpOgHHzEk0MADSmYXQ0FgyEegPXtOFcJqhq1YRsw==" + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.20.1.tgz", + "integrity": "sha512-YiU0s0IzYYC+gWvqD1HzLc46Du1sXpSiwzKb63PACIJr6LfL27VsXSXQvt68EzD3V0D5Bc0vyJTjmMxp0ylQiw==", + "license": "MIT" }, "node_modules/onnxruntime-node": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.19.0.tgz", - "integrity": "sha512-JivYcrZ9H9YPpHnP+5rTVTJjzuD+ZB0TsB0//e2La0ViYNG0hmTvnwFgmRoes6F7E1PyVMpyRftmcl9pnIWsnw==", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.20.1.tgz", + "integrity": "sha512-di/I4HDXRw+FLgq+TyHmQEDd3cEp9iFFZm0r4uJ1Wd7b/WE1VXtKWo8yemex347c6GNF/3Pv86ZfPhIWxORr0w==", "hasInstallScript": true, + "license": "MIT", "os": [ "win32", "darwin", "linux" ], "dependencies": { - "onnxruntime-common": "1.19.0", + "onnxruntime-common": "1.20.1", "tar": "^7.0.1" } }, "node_modules/onnxruntime-web": { - "version": "1.20.0-dev.20240827-1d059b8702", - "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.20.0-dev.20240827-1d059b8702.tgz", - "integrity": "sha512-v1t3TgnOmyzA1lGcUiqp6kZDv7QxO+yJI00vHp8lyjgBrnxkLY+6fqFZpBfm81AH5sRaCdG+HsOmb5Y37TIAyw==", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.20.1.tgz", + "integrity": "sha512-TePF6XVpLL1rWVMIl5Y9ACBQcyCNFThZON/jgElNd9Txb73CIEGlklhYR3UEr1cp5r0rbGI6nDwwrs79g7WjoA==", + "license": "MIT", "dependencies": { "flatbuffers": "^1.12.0", "guid-typescript": "^1.0.9", "long": "^5.2.3", - "onnxruntime-common": "1.20.0-dev.20240827-5d54dc1462", + "onnxruntime-common": "1.20.1", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, - "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { - "version": "1.20.0-dev.20240827-5d54dc1462", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.20.0-dev.20240827-5d54dc1462.tgz", - "integrity": "sha512-oR+xPRD64OI+w9nRLXQi9rEXYZ5W9BhqVi688sUTreU9J6pK182JYblmHjvapCg+Tta6MbkAsr3T1NZHM3tB1g==" - }, "node_modules/openai": { "version": "4.58.1", "resolved": "https://registry.npmjs.org/openai/-/openai-4.58.1.tgz", @@ -10245,7 +10265,8 @@ "node_modules/platform": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", - "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==" + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" }, "node_modules/playwright": { "version": "1.47.0", @@ -10631,6 +10652,7 @@ "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==", "hasInstallScript": true, + "license": "BSD-3-Clause", "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", @@ -12124,6 +12146,7 @@ "version": "7.4.3", "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "license": "ISC", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", @@ -12152,6 +12175,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", + "license": "MIT", "bin": { "mkdirp": "dist/cjs/src/bin.js" }, @@ -12166,6 +12190,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", "engines": { "node": ">=18" } diff --git a/package.json b/package.json index f00619b4966..71148e30166 100644 --- a/package.json +++ b/package.json @@ -63,10 +63,10 @@ "dependencies": { "@aws-sdk/credential-providers": "^3.592.0", "@cliqz/adblocker-playwright": "^1.27.2", - "@gradio/client": "^1.1.1", + "@gradio/client": "^1.8.0", "@huggingface/hub": "^0.5.1", - "@huggingface/inference": "^2.7.0", - "@huggingface/transformers": "^3.0.0-alpha.6", + "@huggingface/inference": "^2.8.1", + "@huggingface/transformers": "^3.1.1", "@iconify-json/bi": "^1.1.21", "@playwright/browser-chromium": "^1.43.1", "@resvg/resvg-js": "^2.6.2", diff --git a/src/lib/server/models.ts b/src/lib/server/models.ts index 43c73dc4084..a5c4ab49b4f 100644 --- a/src/lib/server/models.ts +++ b/src/lib/server/models.ts @@ -129,7 +129,11 @@ async function getChatPromptRender( toolResults, continueMessage, }: ChatTemplateInput) => { - let formattedMessages: { role: string; content: string }[] = messages.map((message) => ({ + let formattedMessages: { + role: string; + content: string; + tool_calls?: { id: string; tool_call_id: string; output: string }[]; + }[] = messages.map((message) => ({ content: message.content, role: message.from, })); @@ -166,7 +170,7 @@ async function getChatPromptRender( if (isHuggingChat && id.startsWith("CohereForAI")) { formattedMessages = [ { - role: m.systemRoleSupported ? "system" : "user", + role: "user", content: "\n\n\n" + toolResults @@ -226,20 +230,6 @@ async function getChatPromptRender( tools = []; } - const chatTemplate = tools?.length ? "tool_use" : undefined; - - const documents = (toolResults ?? []).flatMap((result) => { - if (result.status === ToolResultStatus.Error) { - return [{ title: `Tool "${result.call.name}" error`, text: "\n" + result.message }]; - } - return result.outputs.flatMap((output) => - Object.entries(output).map(([title, text]) => ({ - title: `Tool "${result.call.name}" ${title}`, - text: "\n" + text, - })) - ); - }); - const mappedTools = tools?.map((tool) => { const inputs: Record< @@ -271,9 +261,7 @@ async function getChatPromptRender( const output = tokenizer.apply_chat_template(formattedMessages, { tokenize: false, add_generation_prompt: !continueMessage, - chat_template: chatTemplate, - tools: mappedTools, - documents, + tools: mappedTools.length ? mappedTools : undefined, }); if (typeof output !== "string") { @@ -282,7 +270,6 @@ async function getChatPromptRender( return output; }; - return renderTemplate; } diff --git a/src/lib/server/textGeneration/tools.ts b/src/lib/server/textGeneration/tools.ts index 97d55fe607b..c58a743994c 100644 --- a/src/lib/server/textGeneration/tools.ts +++ b/src/lib/server/textGeneration/tools.ts @@ -184,11 +184,12 @@ export async function* runTools( ? [...formattedMessages.slice(0, -1), fileMsg, ...formattedMessages.slice(-1)] : messages; + let rawText = ""; // do the function calling bits here for await (const output of await endpoint({ messages: formattedMessages, preprompt, - generateSettings: assistant?.generateSettings, + generateSettings: { temperature: 0.1, ...assistant?.generateSettings }, tools: tools.map((tool) => ({ ...tool, inputs: tool.inputs.map((input) => ({ @@ -204,24 +205,44 @@ export async function* runTools( continue; } + if (output.token.text) { + rawText += output.token.text; + } + + // if we dont see a tool call in the first 25 chars, something is going wrong and we abort + if (rawText.length > 25 && !(rawText.includes("```json") || rawText.includes("{"))) { + return []; + } + + // if we see a directly_answer tool call, we skip the rest + if ( + rawText.includes("directly_answer") || + rawText.includes("directlyAnswer") || + rawText.includes("directly-answer") + ) { + return []; + } + // look for a code blocks of ```json and parse them // if they're valid json, add them to the calls array if (output.generated_text) { try { const rawCalls = await extractJson(output.generated_text); const newCalls = rawCalls - .filter(isExternalToolCall) .map((call) => externalToToolCall(call, tools)) .filter((call) => call !== undefined) as ToolCall[]; calls.push(...newCalls); } catch (e) { - logger.error(e, "Error while parsing tool calls, please retry"); + logger.warn( + { rawCall: output.generated_text, error: extractJson }, + "Error while parsing tool calls" + ); // error parsing the calls yield { type: MessageUpdateType.Status, status: MessageUpdateStatus.Error, - message: "Error while parsing tool calls, please retry", + message: "Error while parsing tool calls.", }; } } @@ -239,36 +260,33 @@ export async function* runTools( return toolResults.filter((result): result is ToolResult => result !== undefined); } -const externalToolCall = z.object({ - tool_name: z.string(), - parameters: z.record(z.any()), -}); - -type ExternalToolCall = z.infer; - -function isExternalToolCall(call: unknown): call is ExternalToolCall { - return externalToolCall.safeParse(call).success; -} +function externalToToolCall(call: unknown, tools: Tool[]): ToolCall | undefined { + // Early return if invalid input + if (!isValidCallObject(call)) { + return undefined; + } -function externalToToolCall(call: ExternalToolCall, tools: Tool[]): ToolCall | undefined { - // Convert - to _ since some models insist on using _ instead of - - const tool = tools.find((tool) => toolHasName(call.tool_name, tool)); + const parsedCall = parseExternalCall(call); + if (!parsedCall) return undefined; + const tool = tools.find((tool) => toolHasName(parsedCall.tool_name, tool)); if (!tool) { - logger.debug(`Model requested tool that does not exist: "${call.tool_name}". Skipping tool...`); - return; + logger.debug( + `Model requested tool that does not exist: "${parsedCall.tool_name}". Skipping tool...` + ); + return undefined; } const parametersWithDefaults: Record = {}; for (const input of tool.inputs) { - const value = call.parameters[input.name]; + const value = parsedCall.parameters[input.name]; // Required so ensure it's there, otherwise return undefined if (input.paramType === "required") { if (value === undefined) { logger.debug( - `Model requested tool "${call.tool_name}" but was missing required parameter "${input.name}". Skipping tool...` + `Model requested tool "${parsedCall.tool_name}" but was missing required parameter "${input.name}". Skipping tool...` ); return; } @@ -285,7 +303,41 @@ function externalToToolCall(call: ExternalToolCall, tools: Tool[]): ToolCall | u } return { - name: call.tool_name, + name: parsedCall.tool_name, parameters: parametersWithDefaults, }; } + +// Helper functions +function isValidCallObject(call: unknown): call is Record { + return typeof call === "object" && call !== null; +} + +function parseExternalCall(callObj: Record) { + const nameFields = ["tool_name", "name"] as const; + const parametersFields = ["parameters", "arguments", "parameter_definitions"] as const; + + const groupedCall = { + tool_name: "" as string, + parameters: undefined as Record | undefined, + }; + + for (const name of nameFields) { + if (callObj[name]) { + groupedCall.tool_name = callObj[name] as string; + } + } + + for (const name of parametersFields) { + if (callObj[name]) { + groupedCall.parameters = callObj[name] as Record; + } + } + + return z + .object({ + tool_name: z.string(), + parameters: z.record(z.any()), + }) + .parse(groupedCall); +} diff --git a/src/lib/server/websearch/search/endpoints/bing.ts b/src/lib/server/websearch/search/endpoints/bing.ts index a40804e92a9..20b28ab4999 100644 --- a/src/lib/server/websearch/search/endpoints/bing.ts +++ b/src/lib/server/websearch/search/endpoints/bing.ts @@ -27,8 +27,6 @@ export default async function search(query: string): Promise ); } - console.log(data["webPages"]?.["value"]); - // Adapt the data structure from the Bing response to match the WebSearchSource type const webPages = data["webPages"]?.["value"] ?? []; return webPages.map((page: any) => ({