Merge branch 'master' into feat-request-middleware

.github/workflows/dependabot_auto.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v2.2.0
+        uses: dependabot/fetch-metadata@v2.3.0
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
           skip-commit-verification: true

Makefile

@@ -8,7 +8,7 @@ DETECT_LIBS?=true
 # llama.cpp versions
 GOLLAMA_REPO?=https://github.com/go-skynet/go-llama.cpp
 GOLLAMA_VERSION?=2b57a8ae43e4699d3dc5d1496a1ccd42922993be
-CPPLLAMA_VERSION?=178a7eb952d211b8d4232d5e50ae1b64519172a9
+CPPLLAMA_VERSION?=cae9fb4361138b937464524eed907328731b81f6
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggerganov/whisper.cpp

backend/python/transformers/requirements-cpu.txt

@@ -5,4 +5,4 @@ accelerate
 transformers
 bitsandbytes
 outetts
-sentence-transformers==3.3.1
+sentence-transformers==3.4.0

backend/python/transformers/requirements-cublas11.txt

@@ -6,4 +6,4 @@ accelerate
 transformers
 bitsandbytes
 outetts
-sentence-transformers==3.3.1
+sentence-transformers==3.4.0

backend/python/transformers/requirements-cublas12.txt

@@ -5,4 +5,4 @@ numba==0.60.0
 transformers
 bitsandbytes
 outetts
-sentence-transformers==3.3.1
+sentence-transformers==3.4.0

backend/python/transformers/requirements-hipblas.txt

@@ -7,4 +7,4 @@ numba==0.60.0
 bitsandbytes
 outetts
 bitsandbytes
-sentence-transformers==3.3.1
+sentence-transformers==3.4.0

backend/python/transformers/requirements-intel.txt

@@ -8,4 +8,4 @@ numba==0.60.0
 intel-extension-for-transformers
 bitsandbytes
 outetts
-sentence-transformers==3.3.1
+sentence-transformers==3.4.0

docs/content/docs/advanced/advanced-usage.md

@@ -148,6 +148,9 @@ function:
     no_action_function_name: "" # Function name to call when no action is determined.
     no_action_description_name: "" # Description name for no-action functions.
     response_regex: [] # Regular expressions to match response from
+    argument_regex: [] # Named regular to extract function arguments from the response.
+    argument_regex_key_name: "key" # Name of the named regex capture to capture the key of the function arguments
+	  argument_regex_value_name: "value" # Name of the named regex capture to capture the value of the function arguments
     json_regex_match: [] # Regular expressions to match JSON data when in tool mode
     replace_function_results: [] # Placeholder to replace function call results with arbitrary strings or patterns.
     replace_llm_results: [] # Replace language model results with arbitrary strings or patterns.

docs/content/docs/getting-started/container-images.md

@@ -154,7 +154,7 @@ Images are available with and without python dependencies. Note that images with
 
 Images with `core` in the tag are smaller and do not contain any python dependencies. 
 
-{{< tabs tabTotal="7" >}}
+{{< tabs tabTotal="8" >}}
 {{% tab tabName="Vanilla / CPU Images" %}}
 
 | Description | Quay | Docker Hub                                   |
@@ -236,6 +236,18 @@ Images with `core` in the tag are smaller and do not contain any python dependen
 | Versioned image including FFMpeg, no python | `quay.io/go-skynet/local-ai:{{< version >}}-vulkan-fmpeg-core` | `localai/localai:{{< version >}}-vulkan-fmpeg-core`             |
 {{% /tab %}}
 
+{{% tab tabName="Nvidia Linux for tegra" %}}
+
+These images are compatible with Nvidia ARM64 devices, such as the Jetson Nano, Jetson Xavier NX, and Jetson AGX Xavier. For more information, see the [Nvidia L4T guide]({{%relref "docs/reference/nvidia-l4t" %}}).
+
+| Description | Quay | Docker Hub                                                  |
+| --- | --- |-------------------------------------------------------------|
+| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-nvidia-l4t-arm64-core` | `localai/localai:master-nvidia-l4t-arm64-core`                      |
+| Latest tag | `quay.io/go-skynet/local-ai:latest-nvidia-l4t-arm64-core` | `localai/localai:latest-nvidia-l4t-arm64-core`                 |
+| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-nvidia-l4t-arm64-core` | `localai/localai:{{< version >}}-nvidia-l4t-arm64-core`             |
+
+{{% /tab %}}
+
 {{< /tabs >}}
 
 ## See Also

docs/content/docs/reference/nvidia-l4t.md

@@ -21,15 +21,21 @@ git clone https://github.com/mudler/LocalAI
 
 cd LocalAI
 
-docker build --build-arg SKIP_DRIVERS=true --build-arg BUILD_TYPE=cublas --build-arg BASE_IMAGE=nvcr.io/nvidia/l4t-jetpack:r36.4.0 --build-arg IMAGE_TYPE=core -t localai-orin .
+docker build --build-arg SKIP_DRIVERS=true --build-arg BUILD_TYPE=cublas --build-arg BASE_IMAGE=nvcr.io/nvidia/l4t-jetpack:r36.4.0 --build-arg IMAGE_TYPE=core -t quay.io/go-skynet/local-ai:master-nvidia-l4t-arm64-core .
+```
+
+Otherwise images are available on quay.io and dockerhub:
+
+```bash
+docker pull quay.io/go-skynet/local-ai:master-nvidia-l4t-arm64-core
 ```
 
 ## Usage
 
 Run the LocalAI container on Nvidia ARM64 devices using the following command, where `/data/models` is the directory containing the models:
 
 ```bash
-docker run -e DEBUG=true -p 8080:8080 -v /data/models:/build/models  -ti --restart=always --name local-ai --runtime nvidia --gpus all localai-orin
+docker run -e DEBUG=true -p 8080:8080 -v /data/models:/build/models  -ti --restart=always --name local-ai --runtime nvidia --gpus all quay.io/go-skynet/local-ai:master-nvidia-l4t-arm64-core
 ```
 
 Note: `/data/models` is the directory containing the models. You can replace it with the directory containing your models.

gallery/index.yaml

@@ -839,7 +839,7 @@
       sha256: bac8e8c1d1d9d53cbdb148b8ff9ad378ddb392429207099e85b5aae3a43bff3d
       uri: huggingface://cstr/salamandra-7b-instruct-GGUF/salamandra-7b-instruct.Q4_K_M-f32.gguf
 - &llama32  ## llama3.2
-  url: "github:mudler/LocalAI/gallery/llama3.1-instruct.yaml@master"
+  url: "github:mudler/LocalAI/gallery/llama3.2-quantized.yaml@master"
   icon: https://avatars.githubusercontent.com/u/153379578
   license: llama3.2
   description: |

gallery/llama3.2-quantized.yaml

@@ -0,0 +1,55 @@
+---
+name: "llama3.2-quantized"
+
+config_file: |
+  mmap: true
+  function:
+    disable_no_action: true
+    grammar:
+      disable: true
+    response_regex:
+    - \[(?P<name>\w+)\((?P<arguments>.*)\)\]
+    argument_regex:
+    - (?P<key>[^ '\(=,]+)[='"]+(?P<value>[^=,"']+)['"]?
+  template:
+    chat: |
+      <|begin_of_text|><|start_header_id|>system<|end_header_id|>
+      You are a helpful assistant<|eot_id|><|start_header_id|>user<|end_header_id|>
+      {{.Input }}
+      <|start_header_id|>assistant<|end_header_id|>
+    chat_message: |
+      <|start_header_id|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "tool"}}tool{{else if eq .RoleName "user"}}user{{end}}<|end_header_id|>
+      {{ if .FunctionCall -}}
+      {{ else if eq .RoleName "tool" -}}
+      The Function was executed and the response was:
+      {{ end -}}
+      {{ if .Content -}}
+      {{.Content -}}
+      {{ else if .FunctionCall -}}
+      {{ range .FunctionCall }}
+      [{{.FunctionCall.Name}}({{.FunctionCall.Arguments}})]
+      {{ end }}
+      {{ end -}}
+      <|eot_id|>
+    completion: |
+      {{.Input}}
+    function: |
+      <|start_header_id|>system<|end_header_id|>
+      You are an expert in composing functions. You are given a question and a set of possible functions.
+      Based on the question, you will need to make one or more function/tool calls to achieve the purpose.
+      If none of the functions can be used, point it out. If the given question lacks the parameters required by the function, also point it out. You should only return the function call in tools call sections.
+      If you decide to invoke any of the function(s), you MUST put it in the format as follows:
+      [func_name1(params_name1=params_value1,params_name2=params_value2,...),func_name2(params_name1=params_value1,params_name2=params_value2,...)]
+      You SHOULD NOT include any other text in the response.
+      Here is a list of functions in JSON format that you can invoke.
+      {{toJson .Functions}}
+      <|eot_id|><|start_header_id|>user<|end_header_id|>
+      {{.Input}}
+      <|eot_id|><|start_header_id|>assistant<|end_header_id|>
+  context_size: 8192
+  f16: true
+  stopwords:
+  - <|im_end|>
+  - <dummy32000>
+  - "<|eot_id|>"
+  - <|end_of_text|>

pkg/functions/parse.go

@@ -331,37 +331,36 @@ func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncC
 }
 
 func ParseFunctionCallArgs(functionArguments string, functionConfig FunctionsConfig) string {
-	if len(functionConfig.ArgumentRegex) > 0 {
-		// We use named regexes here to extract the function argument key value pairs and convert this to valid json.
-		// TODO: there might be responses where an object as a value is expected/required. This is currently not handled.
-		args := make(map[string]string)
-
-		agrsRegexKeyName := "key"
-		agrsRegexValueName := "value"
+	if len(functionConfig.ArgumentRegex) == 0 {
+		return functionArguments
+	}
 
-		if functionConfig.ArgumentRegexKey != "" {
-			agrsRegexKeyName = functionConfig.ArgumentRegexKey
-		}
-		if functionConfig.ArgumentRegexValue != "" {
-			agrsRegexValueName = functionConfig.ArgumentRegexValue
-		}
+	// We use named regexes here to extract the function argument key value pairs and convert this to valid json.
+	// TODO: there might be responses where an object as a value is expected/required. This is currently not handled.
+	args := make(map[string]string)
 
-		for _, r := range functionConfig.ArgumentRegex {
-			var respRegex = regexp.MustCompile(r)
-			var nameRange []string = respRegex.SubexpNames()
-			var keyIndex = slices.Index(nameRange, agrsRegexKeyName)
-			var valueIndex = slices.Index(nameRange, agrsRegexValueName)
-			matches := respRegex.FindAllStringSubmatch(functionArguments, -1)
-			for _, match := range matches {
-				args[match[keyIndex]] = match[valueIndex]
-			}
-		}
+	agrsRegexKeyName := "key"
+	agrsRegexValueName := "value"
 
-		jsonBytes, _ := json.Marshal(args)
-		jsonString := string(jsonBytes)
+	if functionConfig.ArgumentRegexKey != "" {
+		agrsRegexKeyName = functionConfig.ArgumentRegexKey
+	}
+	if functionConfig.ArgumentRegexValue != "" {
+		agrsRegexValueName = functionConfig.ArgumentRegexValue
+	}
 
-		return jsonString
-	} else {
-		return functionArguments
+	for _, r := range functionConfig.ArgumentRegex {
+		var respRegex = regexp.MustCompile(r)
+		var nameRange []string = respRegex.SubexpNames()
+		var keyIndex = slices.Index(nameRange, agrsRegexKeyName)
+		var valueIndex = slices.Index(nameRange, agrsRegexValueName)
+		matches := respRegex.FindAllStringSubmatch(functionArguments, -1)
+		for _, match := range matches {
+			args[match[keyIndex]] = match[valueIndex]
+		}
 	}
+
+	jsonBytes, _ := json.Marshal(args)
+
+	return string(jsonBytes)
 }