Skip to content

Commit 45c5875

Browse files
mudlermudler
authored
feat(ui): add audio upload button in chat view (#5526)
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent d5c9c71 commit 45c5875

File tree

5 files changed

+190
-169
lines changed

5 files changed

+190
-169
lines changed

‎core/http/static/chat.js

Lines changed: 158 additions & 160 deletions
Original file line numberDiff line numberDiff line change
@@ -49,20 +49,20 @@ function submitSystemPrompt(event) {
4949
}
5050

5151
var image = "";
52+
var audio = "";
5253

5354
function submitPrompt(event) {
5455
event.preventDefault();
5556

5657
const input = document.getElementById("input").value;
57-
Alpine.store("chat").add("user", input, image);
58+
Alpine.store("chat").add("user", input, image, audio);
5859
document.getElementById("input").value = "";
5960
const systemPrompt = localStorage.getItem("system_prompt");
6061
Alpine.nextTick(() => { document.getElementById('messages').scrollIntoView(false); });
6162
promptGPT(systemPrompt, input);
6263
}
6364

6465
function readInputImage() {
65-
6666
if (!this.files || !this.files[0]) return;
6767

6868
const FR = new FileReader();
@@ -74,35 +74,47 @@ function readInputImage() {
7474
FR.readAsDataURL(this.files[0]);
7575
}
7676

77+
function readInputAudio() {
78+
if (!this.files || !this.files[0]) return;
79+
80+
const FR = new FileReader();
81+
82+
FR.addEventListener("load", function(evt) {
83+
audio = evt.target.result;
84+
});
7785

78-
async function promptGPT(systemPrompt, input) {
79-
const model = document.getElementById("chat-model").value;
80-
// Set class "loader" to the element with "loader" id
81-
//document.getElementById("loader").classList.add("loader");
82-
// Make the "loader" visible
83-
toggleLoader(true);
86+
FR.readAsDataURL(this.files[0]);
87+
}
8488

89+
async function promptGPT(systemPrompt, input) {
90+
const model = document.getElementById("chat-model").value;
91+
// Set class "loader" to the element with "loader" id
92+
//document.getElementById("loader").classList.add("loader");
93+
// Make the "loader" visible
94+
toggleLoader(true);
8595

86-
messages = Alpine.store("chat").messages();
96+
messages = Alpine.store("chat").messages();
8797

88-
// if systemPrompt isn't empty, push it at the start of messages
89-
if (systemPrompt) {
90-
messages.unshift({
91-
role: "system",
92-
content: systemPrompt
93-
});
94-
}
98+
// if systemPrompt isn't empty, push it at the start of messages
99+
if (systemPrompt) {
100+
messages.unshift({
101+
role: "system",
102+
content: systemPrompt
103+
});
104+
}
95105

96-
// loop all messages, and check if there are images. If there are, we need to change the content field
97-
messages.forEach((message) => {
106+
// loop all messages, and check if there are images or audios. If there are, we need to change the content field
107+
messages.forEach((message) => {
108+
if (message.image || message.audio) {
109+
// The content field now becomes an array
110+
message.content = [
111+
{
112+
"type": "text",
113+
"text": message.content
114+
}
115+
]
116+
98117
if (message.image) {
99-
// The content field now becomes an array
100-
message.content = [
101-
{
102-
"type": "text",
103-
"text": message.content
104-
}
105-
]
106118
message.content.push(
107119
{
108120
"type": "image_url",
@@ -111,168 +123,154 @@ function readInputImage() {
111123
}
112124
}
113125
);
114-
115-
// remove the image field
116126
delete message.image;
117127
}
118-
});
119128

120-
// reset the form and the image
121-
image = "";
122-
document.getElementById("input_image").value = null;
123-
document.getElementById("fileName").innerHTML = "";
124-
125-
// if (image) {
126-
// // take the last element content's and add the image
127-
// last_message = messages[messages.length - 1]
128-
// // The content field now becomes an array
129-
// last_message.content = [
130-
// {
131-
// "type": "text",
132-
// "text": last_message.content
133-
// }
134-
// ]
135-
// last_message.content.push(
136-
// {
137-
// "type": "image_url",
138-
// "image_url": {
139-
// "url": image,
140-
// }
141-
// }
142-
// );
143-
// // and we replace it in the messages array
144-
// messages[messages.length - 1] = last_message
145-
146-
// // reset the form and the image
147-
// image = "";
148-
// document.getElementById("input_image").value = null;
149-
// document.getElementById("fileName").innerHTML = "";
150-
// }
151-
152-
// Source: https://stackoverflow.com/a/75751803/11386095
153-
const response = await fetch("v1/chat/completions", {
154-
method: "POST",
155-
headers: {
156-
"Content-Type": "application/json",
157-
},
158-
body: JSON.stringify({
159-
model: model,
160-
messages: messages,
161-
stream: true,
162-
}),
163-
});
164-
165-
if (!response.ok) {
166-
Alpine.store("chat").add(
167-
"assistant",
168-
`<span class='error'>Error: POST /v1/chat/completions ${response.status}</span>`,
169-
);
170-
return;
129+
if (message.audio) {
130+
message.content.push(
131+
{
132+
"type": "audio_url",
133+
"audio_url": {
134+
"url": message.audio,
135+
}
136+
}
137+
);
138+
delete message.audio;
139+
}
171140
}
141+
});
172142

173-
const reader = response.body
174-
?.pipeThrough(new TextDecoderStream())
175-
.getReader();
143+
// reset the form and the files
144+
image = "";
145+
audio = "";
146+
document.getElementById("input_image").value = null;
147+
document.getElementById("input_audio").value = null;
148+
document.getElementById("fileName").innerHTML = "";
149+
150+
// Source: https://stackoverflow.com/a/75751803/11386095
151+
const response = await fetch("v1/chat/completions", {
152+
method: "POST",
153+
headers: {
154+
"Content-Type": "application/json",
155+
},
156+
body: JSON.stringify({
157+
model: model,
158+
messages: messages,
159+
stream: true,
160+
}),
161+
});
176162

177-
if (!reader) {
178-
Alpine.store("chat").add(
179-
"assistant",
180-
`<span class='error'>Error: Failed to decode API response</span>`,
181-
);
182-
return;
183-
}
163+
if (!response.ok) {
164+
Alpine.store("chat").add(
165+
"assistant",
166+
`<span class='error'>Error: POST /v1/chat/completions ${response.status}</span>`,
167+
);
168+
return;
169+
}
184170

185-
// Function to add content to the chat and handle DOM updates efficiently
186-
const addToChat = (token) => {
187-
const chatStore = Alpine.store("chat");
188-
chatStore.add("assistant", token);
189-
// Efficiently scroll into view without triggering multiple reflows
190-
// const messages = document.getElementById('messages');
191-
// messages.scrollTop = messages.scrollHeight;
192-
};
171+
const reader = response.body
172+
?.pipeThrough(new TextDecoderStream())
173+
.getReader();
193174

194-
let buffer = "";
195-
let contentBuffer = [];
175+
if (!reader) {
176+
Alpine.store("chat").add(
177+
"assistant",
178+
`<span class='error'>Error: Failed to decode API response</span>`,
179+
);
180+
return;
181+
}
196182

197-
try {
198-
while (true) {
199-
const { value, done } = await reader.read();
200-
if (done) break;
183+
// Function to add content to the chat and handle DOM updates efficiently
184+
const addToChat = (token) => {
185+
const chatStore = Alpine.store("chat");
186+
chatStore.add("assistant", token);
187+
// Efficiently scroll into view without triggering multiple reflows
188+
// const messages = document.getElementById('messages');
189+
// messages.scrollTop = messages.scrollHeight;
190+
};
201191

202-
buffer += value;
192+
let buffer = "";
193+
let contentBuffer = [];
203194

204-
let lines = buffer.split("\n");
205-
buffer = lines.pop(); // Retain any incomplete line in the buffer
195+
try {
196+
while (true) {
197+
const { value, done } = await reader.read();
198+
if (done) break;
206199

207-
lines.forEach((line) => {
208-
if (line.length === 0 || line.startsWith(":")) return;
209-
if (line === "data: [DONE]") {
210-
return;
211-
}
200+
buffer += value;
201+
202+
let lines = buffer.split("\n");
203+
buffer = lines.pop(); // Retain any incomplete line in the buffer
204+
205+
lines.forEach((line) => {
206+
if (line.length === 0 || line.startsWith(":")) return;
207+
if (line === "data: [DONE]") {
208+
return;
209+
}
212210

213-
if (line.startsWith("data: ")) {
214-
try {
215-
const jsonData = JSON.parse(line.substring(6));
216-
const token = jsonData.choices[0].delta.content;
211+
if (line.startsWith("data: ")) {
212+
try {
213+
const jsonData = JSON.parse(line.substring(6));
214+
const token = jsonData.choices[0].delta.content;
217215

218-
if (token) {
219-
contentBuffer.push(token);
220-
}
221-
} catch (error) {
222-
console.error("Failed to parse line:", line, error);
216+
if (token) {
217+
contentBuffer.push(token);
223218
}
219+
} catch (error) {
220+
console.error("Failed to parse line:", line, error);
224221
}
225-
});
226-
227-
// Efficiently update the chat in batch
228-
if (contentBuffer.length > 0) {
229-
addToChat(contentBuffer.join(""));
230-
contentBuffer = [];
231222
}
232-
}
223+
});
233224

234-
// Final content flush if any data remains
225+
// Efficiently update the chat in batch
235226
if (contentBuffer.length > 0) {
236227
addToChat(contentBuffer.join(""));
228+
contentBuffer = [];
237229
}
238-
239-
// Highlight all code blocks once at the end
240-
hljs.highlightAll();
241-
} catch (error) {
242-
console.error("An error occurred while reading the stream:", error);
243-
Alpine.store("chat").add(
244-
"assistant",
245-
`<span class='error'>Error: Failed to process stream</span>`,
246-
);
247-
} finally {
248-
// Perform any cleanup if necessary
249-
reader.releaseLock();
250230
}
251231

252-
// Remove class "loader" from the element with "loader" id
253-
toggleLoader(false);
232+
// Final content flush if any data remains
233+
if (contentBuffer.length > 0) {
234+
addToChat(contentBuffer.join(""));
235+
}
254236

255-
// scroll to the bottom of the chat
256-
document.getElementById('messages').scrollIntoView(false)
257-
// set focus to the input
258-
document.getElementById("input").focus();
237+
// Highlight all code blocks once at the end
238+
hljs.highlightAll();
239+
} catch (error) {
240+
console.error("An error occurred while reading the stream:", error);
241+
Alpine.store("chat").add(
242+
"assistant",
243+
`<span class='error'>Error: Failed to process stream</span>`,
244+
);
245+
} finally {
246+
// Perform any cleanup if necessary
247+
reader.releaseLock();
259248
}
260249

261-
document.getElementById("system_prompt").addEventListener("submit", submitSystemPrompt);
250+
// Remove class "loader" from the element with "loader" id
251+
toggleLoader(false);
262252

263-
document.getElementById("prompt").addEventListener("submit", submitPrompt);
253+
// scroll to the bottom of the chat
254+
document.getElementById('messages').scrollIntoView(false)
255+
// set focus to the input
264256
document.getElementById("input").focus();
265-
document.getElementById("input_image").addEventListener("change", readInputImage);
257+
}
266258

267-
storesystemPrompt = localStorage.getItem("system_prompt");
268-
if (storesystemPrompt) {
269-
document.getElementById("systemPrompt").value = storesystemPrompt;
270-
} else {
271-
document.getElementById("systemPrompt").value = null;
272-
}
259+
document.getElementById("system_prompt").addEventListener("submit", submitSystemPrompt);
260+
document.getElementById("prompt").addEventListener("submit", submitPrompt);
261+
document.getElementById("input").focus();
262+
document.getElementById("input_image").addEventListener("change", readInputImage);
263+
document.getElementById("input_audio").addEventListener("change", readInputAudio);
264+
265+
storesystemPrompt = localStorage.getItem("system_prompt");
266+
if (storesystemPrompt) {
267+
document.getElementById("systemPrompt").value = storesystemPrompt;
268+
} else {
269+
document.getElementById("systemPrompt").value = null;
270+
}
273271

274-
marked.setOptions({
275-
highlight: function (code) {
276-
return hljs.highlightAuto(code).value;
277-
},
278-
});
272+
marked.setOptions({
273+
highlight: function (code) {
274+
return hljs.highlightAuto(code).value;
275+
},
276+
});

0 commit comments

Comments
 (0)