Skip to content

Commit 840dfb0

Browse files
ConcedoConcedo
Concedo
authored and
Concedo
committed
openai sse streaming mode
1 parent 33bc9f5 commit 840dfb0

File tree

1 file changed

+216
-60
lines changed

1 file changed

+216
-60
lines changed

‎index.html

+216-60
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
-->
1313

1414
<script>
15-
const LITEVER = 159;
15+
const LITEVER = 160;
1616
const urlParams = new URLSearchParams(window.location.search);
1717
const localflag = urlParams.get('local');
1818
const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
@@ -3540,6 +3540,170 @@
35403540
}
35413541
}
35423542

3543+
function oai_api_sync_req(targetep,oai_payload,oaiheaders)
3544+
{
3545+
fetch(targetep, {
3546+
method: 'POST',
3547+
headers: oaiheaders,
3548+
body: JSON.stringify(oai_payload),
3549+
referrerPolicy: 'no-referrer',
3550+
})
3551+
.then((response) => response.json())
3552+
.then((data) => {
3553+
console.log("sync finished response: " + JSON.stringify(data));
3554+
if (custom_oai_key != "" && data.choices != null && data.choices.length > 0) {
3555+
let dch = data.choices[0];
3556+
if (dch.text) {
3557+
synchro_polled_response = dch.text;
3558+
}
3559+
else if (dch.message) {
3560+
synchro_polled_response = dch.message.content;
3561+
3562+
if(localsettings.opmode==1 && gametext_arr.length>0 && synchro_polled_response!="")
3563+
{
3564+
synchro_polled_response = cleanup_story_completion(synchro_polled_response);
3565+
}
3566+
}
3567+
else {
3568+
console.error("Error, unknown OAI response");
3569+
clear_poll_flags();
3570+
render_gametext();
3571+
msgbox("Error, unknown OAI response");
3572+
}
3573+
}
3574+
else {
3575+
//error occurred, maybe captcha failed
3576+
console.error("error occurred in OAI generation");
3577+
clear_poll_flags();
3578+
render_gametext();
3579+
msgbox("Error occurred during text generation: " + formatError(data));
3580+
}
3581+
})
3582+
.catch((error) => {
3583+
console.error('Error:', error);
3584+
clear_poll_flags();
3585+
render_gametext();
3586+
msgbox("Error while submitting prompt: " + error);
3587+
});
3588+
}
3589+
3590+
function oai_api_stream_sse(sub_endpt,submit_payload,submit_headers)
3591+
{
3592+
synchro_pending_stream = "";
3593+
let reqOpt =
3594+
{method: 'POST',
3595+
headers: submit_headers,
3596+
body: JSON.stringify(submit_payload)};
3597+
if(globalabortcontroller)
3598+
{
3599+
reqOpt.signal = globalabortcontroller.signal;
3600+
}
3601+
fetch(sub_endpt, reqOpt)
3602+
.then(x => {
3603+
if(x.ok)
3604+
{
3605+
return x;
3606+
}else{
3607+
return x.text().then(errdat => {
3608+
throw new Error('Error while SSE streaming: ' + errdat);
3609+
return null;
3610+
}).catch(err => {
3611+
throw new Error('Error while SSE streaming: ' + (x.statusText) + '\n' + err);
3612+
return null;
3613+
});
3614+
}
3615+
})
3616+
.then(resp => {
3617+
resp.body
3618+
.pipeThrough(new TextDecoderStream())
3619+
.pipeThrough(new TransformStream({
3620+
start(ctrl) {
3621+
ctrl.buf = '';
3622+
},
3623+
transform(chunk, ctrl) {
3624+
ctrl.buf += chunk;
3625+
let evs = [];
3626+
let m;
3627+
while ((m = /^data: (.*)\n\n/m.exec(ctrl.buf)) !== null) {
3628+
try{evs.push({data: JSON.parse(m[1])});} catch (e) {}
3629+
ctrl.buf = ctrl.buf.substring(m.index + m[0].length);
3630+
}
3631+
if (evs.length) {
3632+
ctrl.enqueue(evs);
3633+
}
3634+
}
3635+
}))
3636+
.pipeTo(new WritableStream({
3637+
write(chunk) {
3638+
let was_empty = (synchro_pending_stream=="");
3639+
//cut stream if aborted
3640+
if(pending_response_id && pending_response_id != "-1" && pending_response_id != "")
3641+
{
3642+
for (let event of chunk) {
3643+
if (event.data && event.data.choices && event.data.choices.length>0) {
3644+
if(event.data.choices[0].text)
3645+
{
3646+
synchro_pending_stream += event.data.choices[0].text;
3647+
}else if(event.data.choices[0].delta && event.data.choices[0].delta.content)
3648+
{
3649+
synchro_pending_stream += event.data.choices[0].delta.content;
3650+
}
3651+
3652+
if(event.data.choices[0].finish_reason=="stop")
3653+
{
3654+
last_stop_reason = "stop";
3655+
}
3656+
}
3657+
}
3658+
}
3659+
if(was_empty && synchro_pending_stream!="")
3660+
{
3661+
render_gametext(false);
3662+
}
3663+
else
3664+
{
3665+
update_pending_stream_displays();
3666+
}
3667+
},
3668+
close() { //end of stream
3669+
synchro_polled_response = synchro_pending_stream;
3670+
let need_clean_output = (synchro_polled_response!="" && localsettings.opmode==1 && gametext_arr.length>0 && document.getElementById("useoaichatcompl").checked);
3671+
if(need_clean_output)
3672+
{
3673+
synchro_polled_response = cleanup_story_completion(synchro_polled_response);
3674+
}
3675+
synchro_pending_stream = "";
3676+
poll_pending_response();
3677+
//handle gen failures
3678+
if(resp.status==503)
3679+
{
3680+
msgbox("Error while submitting prompt: Server appears to be busy.");
3681+
}
3682+
},
3683+
abort(error) {
3684+
console.error('Error:', error);
3685+
if(error.name!="AbortError") //aborts are silent. slightly diff logic
3686+
{
3687+
flush_streaming_text();
3688+
msgbox("Error while submitting prompt: " + error);
3689+
}
3690+
clear_poll_flags();
3691+
render_gametext();
3692+
},
3693+
}));
3694+
})
3695+
.catch((error) => {
3696+
console.error('Error:', error);
3697+
if(error.name!="AbortError") //aborts are silent. slightly diff logic
3698+
{
3699+
flush_streaming_text();
3700+
msgbox("Error while submitting prompt: " + error);
3701+
}
3702+
clear_poll_flags();
3703+
render_gametext();
3704+
});
3705+
}
3706+
35433707
function kobold_api_stream_sse(sub_endpt,submit_payload)
35443708
{
35453709
synchro_pending_stream = "";
@@ -3572,7 +3736,7 @@
35723736
ctrl.buf += chunk;
35733737
let evs = [];
35743738
let m;
3575-
while ((m = /^event: (.*)\ndata: (.*)\n\n/.exec(ctrl.buf)) !== null) {
3739+
while ((m = /^event: (.*)\ndata: (.*)\n\n/m.exec(ctrl.buf)) !== null) {
35763740
evs.push({event: m[1], data: JSON.parse(m[2])});
35773741
ctrl.buf = ctrl.buf.substring(m.index + m[0].length);
35783742
}
@@ -5152,19 +5316,21 @@
51525316
}
51535317
}
51545318

5319+
function is_browser_supports_sse()
5320+
{
5321+
return (self.TransformStream!=null && self.TextDecoderStream!=null && self.WritableStream!=null);
5322+
}
51555323
function is_using_custom_ep()
51565324
{
51575325
return (custom_oai_key!=""||custom_kobold_endpoint!=""||custom_claude_key!=""||custom_palm_key!=""||custom_cohere_key!="");
51585326
}
5159-
51605327
function is_using_kcpp_with_streaming()
51615328
{
51625329
return (custom_kobold_endpoint!="" && koboldcpp_version && koboldcpp_version!="" && compare_version_str(koboldcpp_version, "1.30") >= 0);
51635330
}
51645331
function is_using_kcpp_with_sse() //need 1.39 for multibyte fix
51655332
{
5166-
let browsersupported = (self.TransformStream!=null && self.TextDecoderStream!=null && self.WritableStream!=null);
5167-
return (browsersupported && custom_kobold_endpoint!="" && koboldcpp_version && koboldcpp_version!="" && compare_version_str(koboldcpp_version, "1.40") >= 0);
5333+
return (is_browser_supports_sse() && custom_kobold_endpoint!="" && koboldcpp_version && koboldcpp_version!="" && compare_version_str(koboldcpp_version, "1.40") >= 0);
51685334
}
51695335
function is_using_kcpp_with_mirostat()
51705336
{
@@ -7880,9 +8046,9 @@
78808046
let ddval = document.getElementById("customapidropdown").value;
78818047
switch(ddval)
78828048
{
7883-
case 3:
8049+
case "3":
78848050
return document.getElementById("custom_openrouter_model");
7885-
case 7:
8051+
case "7":
78868052
return document.getElementById("custom_mistralai_model");
78878053
default:
78888054
return document.getElementById("custom_oai_model");
@@ -11750,15 +11916,36 @@
1175011916

1175111917
function cleanup_story_completion(resp)
1175211918
{
11753-
if(!gametext_arr[gametext_arr.length-1].endsWith(" ") && !gametext_arr[gametext_arr.length-1].endsWith("\n"))
11919+
if(gametext_arr.length>0)
1175411920
{
11755-
if(/^\.\.\.[a-zA-Z0-9]/.test(resp))
11921+
//fix duplicate sentences
11922+
const sentenceEndings = /[.!?]/g;
11923+
let lastsentences = gametext_arr[gametext_arr.length-1].split(sentenceEndings);
11924+
lastsentences = lastsentences.map(lastsentences => lastsentences.trim()); //remove whitespace
11925+
lastsentences = lastsentences.filter(lastsentences => lastsentences.length > 0);
11926+
if(lastsentences.length>0)
1175611927
{
11757-
resp = resp.slice(3);
11928+
let lastsentence = lastsentences[lastsentences.length - 1];
11929+
if(lastsentence.length>10 && resp.trim().startsWith(lastsentence)) //only match if its long enough and matches verbatim
11930+
{
11931+
let foundindex = resp.indexOf(lastsentence);
11932+
if (foundindex !== -1 && foundindex<5) {
11933+
resp = resp.substring(foundindex+lastsentence.length); //remove duplicated part
11934+
}
11935+
}
1175811936
}
11759-
if (/^[^!\"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~ \t\r\n\f\v]/.test(resp)) // List of common punctuation and whitespace
11937+
11938+
//fix response lacking space
11939+
if(!gametext_arr[gametext_arr.length-1].endsWith(" ") && !gametext_arr[gametext_arr.length-1].endsWith("\n"))
1176011940
{
11761-
resp = " "+resp;
11941+
if(/^\.\.\.[a-zA-Z0-9]/.test(resp))
11942+
{
11943+
resp = resp.slice(3);
11944+
}
11945+
if (/^[^!\"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~ \t\r\n\f\v]/.test(resp)) // List of common punctuation and whitespace
11946+
{
11947+
resp = " "+resp;
11948+
}
1176211949
}
1176311950
}
1176411951
return resp;
@@ -11957,49 +12144,15 @@
1195712144
oaiheaders["HTTP-Referer"] = "https://lite.koboldai.net";
1195812145
}
1195912146

11960-
fetch(targetep, {
11961-
method: 'POST',
11962-
headers: oaiheaders,
11963-
body: JSON.stringify(oai_payload),
11964-
referrerPolicy: 'no-referrer',
11965-
})
11966-
.then((response) => response.json())
11967-
.then((data) => {
11968-
console.log("sync finished response: " + JSON.stringify(data));
11969-
if (custom_oai_key != "" && data.choices != null && data.choices.length > 0) {
11970-
let dch = data.choices[0];
11971-
if (dch.text) {
11972-
synchro_polled_response = dch.text;
11973-
}
11974-
else if (dch.message) {
11975-
synchro_polled_response = dch.message.content;
11976-
11977-
if(localsettings.opmode==1 && gametext_arr.length>0 && synchro_polled_response!="")
11978-
{
11979-
synchro_polled_response = cleanup_story_completion(synchro_polled_response);
11980-
}
11981-
}
11982-
else {
11983-
console.error("Error, unknown OAI response");
11984-
clear_poll_flags();
11985-
render_gametext();
11986-
msgbox("Error, unknown OAI response");
11987-
}
11988-
}
11989-
else {
11990-
//error occurred, maybe captcha failed
11991-
console.error("error occurred in OAI generation");
11992-
clear_poll_flags();
11993-
render_gametext();
11994-
msgbox("Error occurred during text generation: " + formatError(data));
11995-
}
11996-
})
11997-
.catch((error) => {
11998-
console.error('Error:', error);
11999-
clear_poll_flags();
12000-
render_gametext();
12001-
msgbox("Error while submitting prompt: " + error);
12002-
});
12147+
if(is_browser_supports_sse() && document.getElementById("oaistreaming").checked)
12148+
{
12149+
oai_payload.stream = true;
12150+
oai_api_stream_sse(targetep,oai_payload,oaiheaders);
12151+
}
12152+
else
12153+
{
12154+
oai_api_sync_req(targetep,oai_payload,oaiheaders);
12155+
}
1200312156
}
1200412157
else if (custom_claude_key != "")//handle for Claude
1200512158
{
@@ -17105,11 +17258,14 @@
1710517258
</select>
1710617259
<button type="button" class="btn btn-primary" style="display:inline;width:105px;" id="oaifetchlist" onclick="oai_fetch_models()">Fetch List</button>
1710717260
<button type="button" class="btn btn-primary" style="display:inline;width:105px;" id="oaiusecustom" onclick="select_custom_oai_model()">Use Custom</button>
17108-
<input type="checkbox" id="oaiaddversion" onchange="" checked>
17109-
<div class="box-label" title="Add endpoint version">Add Endpoint Version</div>
17110-
<input type="checkbox" id="useoaichatcompl" onchange="toggleoaichatcompl()">
17111-
<div class="box-label" id="useoaichatcompllabel">Use ChatCompletions API</div>
17112-
17261+
<div style="display:inline-flex">
17262+
<div><input type="checkbox" id="oaiaddversion" title="Add Endpoint Version Number" onchange="" checked>
17263+
<div class="box-label">Add Version Num</div></div>
17264+
<div><input type="checkbox" id="oaistreaming" title="Enable SSE Streaming" onchange="">
17265+
<div class="box-label">Streaming</div></div>
17266+
<div><input type="checkbox" id="useoaichatcompl" title="Use ChatCompletions API" onchange="toggleoaichatcompl()">
17267+
<div class="box-label" id="useoaichatcompllabel">ChatCompletions API</div></div>
17268+
</div>
1711317269
<span id="useoaichatcomplbox" class="hidden" onload="toggleoaichatcompl();">
1711417270
<br>
1711517271
Main Message Role:

0 commit comments

Comments
 (0)