Merge branch 'concedo_experimental' into croco_nex_0

Author: Nexesenex

Diff for: ggml/src/ggml-metal/ggml-metal.metal

@@ -373,24 +373,33 @@ void dequantize_q5_K(device const block_q5_K *xb, short il, thread type4x4 & reg
 template <typename type4x4>
 void dequantize_q6_K(device const block_q6_K *xb, short il, thread type4x4 & reg) {
     const half d_all = xb->d;
-    device const uint8_t * ql = (device const uint8_t *)xb->ql;
-    device const uint8_t * qh = (device const uint8_t *)xb->qh;
+    device const uint16_t * ql = (device const uint16_t *)xb->ql;
+    device const uint16_t * qh = (device const uint16_t *)xb->qh;
     device const int8_t * scales = (device const int8_t *)xb->scales;
 
-    ql = ql + 64*(il/8) + 32*((il/2)&1) + 16*(il&1);
-    qh = qh + 32*(il/8) + 16*(il&1);
+    ql = ql + 32*(il/8) + 16*((il/2)&1) + 8*(il&1);
+    qh = qh + 16*(il/8) + 8*(il&1);
     float sc = scales[(il%2) + 2 * ((il/2))];
     il = (il/2) & 3;
 
-    const uint16_t  kmask1 = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3);
-    const uint16_t  kmask2 = il>1 ? 0xF0              : 0x0F;
-    const float       coef = il>1 ? 1.f/16.f          : 1.f;
+    const uint32_t kmask1 = il>1 ? (il>2 ? 0xC0C0C0C0 : 0x30303030) : (il>0 ? 0x0C0C0C0C : 0x03030303);
+    const uint32_t kmask2 = il>1 ? 0xF0F0F0F0                       : 0x0F0F0F0F;
     const float ml = d_all * sc * 32.f;
-    const float dl = d_all * sc * coef;
-    for (int i = 0; i < 16; ++i) {
-        const half q = il&1 ? ((ql[i] & kmask2) | ((qh[i] & kmask1) << 2))
-                            : ((ql[i] & kmask2) | ((qh[i] & kmask1) << 4));
-        reg[i/4][i%4] = dl * q - ml;
+    const float dl0 = d_all * sc;
+    const float dl1 = dl0 / 256.f;
+    const float dl2 = dl0 / (256.f * 256.f);
+    const float dl3 = dl0 / (256.f * 256.f * 256.f);
+    const uint8_t shr_h = il>2 ? 2 : 0;
+    const uint8_t shl_h = il>1 ? 0 : (il>0 ? 2 : 4);
+    const uint8_t shr_l = il>1 ? 4 : 0;
+    for (int i = 0; i < 4; ++i) {
+        const uint32_t  low = (ql[2*i] | (uint32_t)(ql[2*i+1] << 16)) & kmask2;
+        const uint32_t high = (qh[2*i] | (uint32_t)(qh[2*i+1] << 16)) & kmask1;
+        const uint32_t q = ((high << shl_h) >> shr_h) | (low >> shr_l);
+        reg[i][0] = dl0 *  ((half)(q & 0xFF))       - ml;
+        reg[i][1] = dl1 * ((float)(q & 0xFF00))     - ml;
+        reg[i][2] = dl2 * ((float)(q & 0xFF0000))   - ml;
+        reg[i][3] = dl3 * ((float)(q & 0xFF000000)) - ml;
     }
 }
 

Diff for: klite.embd

@@ -12,7 +12,7 @@ Current version indicated by LITEVER below.
 -->
 
 <script>
-	const LITEVER = 214;
+	const LITEVER = 215;
 	const urlParams = new URLSearchParams(window.location.search);
 	var localflag = true;
 	const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
@@ -11913,7 +11913,7 @@ Current version indicated by LITEVER below.
 		});
 	}
 
-	function clear_poll_flags()
+	function restore_retried_text()
 	{
 		if(retry_in_progress)
 		{
@@ -11928,6 +11928,10 @@ Current version indicated by LITEVER below.
 				}
 			}
 		}
+	}
+	function clear_poll_flags()
+	{
+		restore_retried_text();
 		pending_response_id = "";
 		poll_in_progress = false;
 		synchro_polled_response = null;
@@ -16484,6 +16488,7 @@ Current version indicated by LITEVER below.
 								handle_incoming_text(gentxt, genworker, genmdl, genkudos);
 							}
 						}else{
+							restore_retried_text();
 							retry_preserve_last = false;
 						}
 						synchro_polled_response = null;
@@ -16544,7 +16549,13 @@ Current version indicated by LITEVER below.
 												}
 												else {
 													last_reply_was_empty = (gentxt=="" || gentxt.trim()=="");
+													let was_retry_in_progress = retry_in_progress;
 													handle_incoming_text(gentxt, genworker, genmdl, genkudos);
+													if (gentxt=="" && was_retry_in_progress)
+													{
+														retry_in_progress = was_retry_in_progress;
+														restore_retried_text(); //horde only: this handles the case when the retry returned empty text, we restore the old text
+													}
 												}
 											}
 											render_gametext();

Diff for: koboldcpp.py

@@ -55,7 +55,7 @@
 # dry_seq_break_max = 128
 
 # global vars
-KcppVersion = "1.84000"
+KcppVersion = "1.84200"
 LcppVersion = "b4722"
 CudaSpecifics = "Cu128_Ar86_SMC2_DmmvX32Y1"
 ReleaseDate = "2025/02/15"
@@ -106,6 +106,7 @@
 websearch_lastresponse = []
 preloaded_story = None
 chatcompl_adapter = None
+chatcompl_adapter_list = None #if using autoguess, will populate this will potential adapters
 embedded_kailite = None
 embedded_kcpp_docs = None
 embedded_kcpp_sdui = None
@@ -3093,7 +3094,7 @@ def do_GET(self):
             opts = []
             if args.admin and args.admindir and os.path.exists(args.admindir) and self.check_header_password(args.adminpassword):
                 dirpath = os.path.abspath(args.admindir)
-                opts = [f for f in sorted(os.listdir(dirpath)) if f.endswith(".kcpps") and os.path.isfile(os.path.join(dirpath, f))]
+                opts = [f for f in sorted(os.listdir(dirpath)) if (f.endswith(".kcpps") or f.endswith(".kcppt")) and os.path.isfile(os.path.join(dirpath, f))]
             response_body = (json.dumps(opts).encode())
 
         elif self.path.endswith(('/api/extra/perf')):
@@ -3465,7 +3466,7 @@ def do_POST(self):
                 if targetfile and targetfile!="":
                     dirpath = os.path.abspath(args.admindir)
                     targetfilepath = os.path.join(dirpath, targetfile)
-                    opts = [f for f in os.listdir(dirpath) if f.endswith(".kcpps") and os.path.isfile(os.path.join(dirpath, f))]
+                    opts = [f for f in os.listdir(dirpath) if (f.endswith(".kcpps") or f.endswith(".kcppt")) and os.path.isfile(os.path.join(dirpath, f))]
                     if targetfile in opts and os.path.exists(targetfilepath):
                         print(f"Admin: Received request to reload config to {targetfile}")
                         global_memory["restart_target"] = targetfile
@@ -5384,7 +5385,7 @@ def make_url_request_horde(url, data, method='POST',addmykey=False):
             "name": worker_name,
             "models": [friendlymodelname],
             "max_length": maxhordelen,
-            "max_context_length": maxhordectx,
+            "max_context_length": min(maxctx,maxhordectx),
             "priority_usernames": [],
             "softprompts": [],
             "bridge_agent": BRIDGE_AGENT,
@@ -5697,7 +5698,7 @@ def sanitize_string(input_string):
     sanitized_string = re.sub( r'[^\w\d\.\-_]', '', input_string)
     return sanitized_string
 
-def downloader_internal(input_url, output_filename, capture_output, min_file_size=64): #64 bytes required by default
+def downloader_internal(input_url, output_filename, capture_output, min_file_size=64): # 64 bytes required by default
     import shutil
     import subprocess
     import os
@@ -5711,20 +5712,44 @@ def downloader_internal(input_url, output_filename, capture_output, min_file_siz
         return output_filename
     print(f"Downloading {input_url}", flush=True)
     dl_success = False
-    if shutil.which("aria2c") is not None:
-        rc = subprocess.run(f"aria2c -x 16 -s 16 --summary-interval=30 --console-log-level=error --log-level=error --download-result=default --allow-overwrite=true --file-allocation=none -o {output_filename} {input_url}", shell=True, capture_output=capture_output, text=True, check=True, encoding='utf-8')
-        dl_success = (rc.returncode==0 and os.path.exists(output_filename) and os.path.getsize(output_filename) > min_file_size)
-    if not dl_success and shutil.which("curl") is not None:
-        rc = subprocess.run(f"curl -fLo {output_filename} {input_url}", shell=True, capture_output=capture_output, text=True, check=True, encoding='utf-8')
-        dl_success = (rc.returncode==0 and os.path.exists(output_filename) and os.path.getsize(output_filename) > min_file_size)
-    if not dl_success and shutil.which("wget") is None:
-        rc = subprocess.run(f"wget -O {output_filename} {input_url}", shell=True, capture_output=capture_output, text=True, check=True, encoding='utf-8')
-        dl_success = (rc.returncode==0 and os.path.exists(output_filename) and os.path.getsize(output_filename) > min_file_size)
+
+    try:
+        if shutil.which("aria2c") is not None:
+            rc = subprocess.run(
+                f"aria2c -x 16 -s 16 --summary-interval=30 --console-log-level=error --log-level=error --download-result=default --allow-overwrite=true --file-allocation=none -o {output_filename} {input_url}",
+                shell=True, capture_output=capture_output, text=True, check=True, encoding='utf-8'
+            )
+            dl_success = (rc.returncode == 0 and os.path.exists(output_filename) and os.path.getsize(output_filename) > min_file_size)
+    except subprocess.CalledProcessError as e:
+        print(f"aria2c failed: {e}")
+
+    try:
+        if not dl_success and shutil.which("curl") is not None:
+            rc = subprocess.run(
+                f"curl -fLo {output_filename} {input_url}",
+                shell=True, capture_output=capture_output, text=True, check=True, encoding='utf-8'
+            )
+            dl_success = (rc.returncode == 0 and os.path.exists(output_filename) and os.path.getsize(output_filename) > min_file_size)
+    except subprocess.CalledProcessError as e:
+        print(f"curl failed: {e}")
+
+    try:
+        if not dl_success and shutil.which("wget") is not None:
+            rc = subprocess.run(
+                f"wget -O {output_filename} {input_url}",
+                shell=True, capture_output=capture_output, text=True, check=True, encoding='utf-8'
+            )
+            dl_success = (rc.returncode == 0 and os.path.exists(output_filename) and os.path.getsize(output_filename) > min_file_size)
+    except subprocess.CalledProcessError as e:
+        print(f"wget failed: {e}")
+
     if not dl_success:
-        print("Could not find suitable download software, please install aria2 or curl.")
+        print("Could not find suitable download software, or all download methods failed. Please install aria2, curl, or wget.")
         return None
+
     return output_filename
 
+
 def download_model_from_url(url, permitted_types=[".gguf",".safetensors", ".ggml", ".bin"], min_file_size=64):
     if url and url!="":
         if url.endswith("?download=true"):
@@ -5789,6 +5814,9 @@ def main(launch_args):
         # print("Python version: " + sys.version)
     if args.debugmode != 1:
         showdebug = False #not shared with child process!
+    if args.debugmode >= 1:
+        print("Debug Mode is Enabled!")
+        args.quiet = False # verbose outputs
 
     try:
         delete_old_pyinstaller()  #perform some basic cleanup of old temporary directories
@@ -5991,7 +6019,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
 
     # try to read chat completions adapter
     if args.chatcompletionsadapter:
-        global chatcompl_adapter
+        global chatcompl_adapter, chatcompl_adapter_list
         ccadapter_path = None
         canload = False
         adapt_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'kcpp_adapters')
@@ -6031,6 +6059,9 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
             print("Chat Completions Adapter Loaded")
         else:
             print("Warning: Chat Completions Adapter invalid or not found.")
+        if (chatcompl_adapter is not None and isinstance(chatcompl_adapter, list)):
+            chatcompl_adapter_list = chatcompl_adapter
+            chatcompl_adapter = None
 
     # handle model downloads if needed
     if args.model_param and args.model_param!="":
@@ -6238,23 +6269,20 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
             exitcounter = 999
             exit_with_error(3,"Could not load text model: " + modelname)
 
-    if (chatcompl_adapter is not None and isinstance(chatcompl_adapter, list)):
+    if (chatcompl_adapter_list is not None and isinstance(chatcompl_adapter_list, list)):
         # The chat completions adapter is a list that needs derivation from chat templates
         # Try to derive chat completions adapter from chat template, now that we have the model loaded
         if not args.nomodel and args.model_param:
             ctbytes = handle.get_chat_template()
             chat_template = ctypes.string_at(ctbytes).decode("UTF-8","ignore")
-            candidates = chatcompl_adapter
             if chat_template != "":
-                for entry in candidates:
+                for entry in chatcompl_adapter_list:
                     if all(s in chat_template for s in entry['search']):
                         print(f"Chat completion heuristic: {entry['name']}")
                         chatcompl_adapter = entry['adapter']
                         break
             if chatcompl_adapter is None:
                 print("Chat template heuristics failed to identify chat completions format. Alpaca will be used.")
-        else:
-            chatcompl_adapter = None #if no text model loaded, erase the list.
 
     #handle loading image model
     if args.sdmodel and args.sdmodel!="":
@@ -6445,7 +6473,10 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
                 print(f"StableUI is available at {endpoint_url}/sdui/")
         global_memory["load_complete"] = True
     if args.launch:
-        LaunchWebbrowser(endpoint_url,"--launch was set, but could not launch web browser automatically.")
+        def launch_browser_thread():
+            LaunchWebbrowser(endpoint_url,"--launch was set, but could not launch web browser automatically.")
+        browser_thread = threading.Timer(2, launch_browser_thread) #2 second delay
+        browser_thread.start()
 
     if args.hordekey and args.hordekey!="":
         if args.hordeworkername and args.hordeworkername!="":