Skip to content

Commit 8b992dd

Browse files
lmg-anonlmg-anon
committed
Add option to preview predictions
1 parent b14ac75 commit 8b992dd

File tree

1 file changed

+150
-20
lines changed

1 file changed

+150
-20
lines changed

‎mikupad.html

+150-20
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,32 @@
318318
outline: 1px solid var(--color-base-50);
319319
outline-offset: 1px;
320320
}
321+
#prompt-container #prompt-overlay > .preview {
322+
color: var(--color-dark);
323+
opacity: 40%;
324+
}
325+
html.serif-dark #prompt-container #prompt-overlay > .preview,
326+
html.monospace-dark #prompt-container #prompt-overlay > .preview,
327+
html.nockoffAI #prompt-container #prompt-overlay > .preview {
328+
color: var(--color-light);
329+
opacity: 30%;
330+
}
331+
#prompt-container #prompt-overlay > .nudge {
332+
outline: 1px solid var(--color-dark);
333+
margin-left: 5px;
334+
padding-bottom: 2px;
335+
padding: 0 4px;
336+
font-size: calc(11px * var(--font-size-multiplier));
337+
background-color: transparent;
338+
border-radius: 3px;
339+
position: relative;
340+
top: -2px;
341+
}
342+
html.serif-dark #prompt-container #prompt-overlay > .nudge,
343+
html.monospace-dark #prompt-container #prompt-overlay > .nudge,
344+
html.nockoffAI #prompt-container #prompt-overlay > .nudge {
345+
border: 1px solid var(--color-light);
346+
}
321347

322348
#probs {
323349
position: absolute;
@@ -5488,6 +5514,8 @@
54885514
grammar: '',
54895515
chatAPI: false,
54905516
tokenStreaming: true,
5517+
promptPreview: false,
5518+
promptPreviewTokens: 20,
54915519
};
54925520

54935521
function joinPrompt(prompt) {
@@ -5690,6 +5718,7 @@
56905718
const sessionReconnectTimer = useRef();
56915719
const useScrollSmoothing = useRef(true);
56925720
const hordeTaskId = useRef();
5721+
const promptPreviewElement = useRef();
56935722
const [templates, setTemplates] = useDBTemplates(defaultPresets.instructTemplates);
56945723
const [templateReplacements, setTemplateReplacements] = useState(false);
56955724
const [templatesImport, setTemplatesImport] = useState(false);
@@ -5775,6 +5804,10 @@
57755804
const [hordeProcessing, setHordeProcessing] = useState(false);
57765805
const [useChatAPI, setUseChatAPI] = useSessionState('chatAPI', defaultPresets.chatAPI);
57775806
const [useTokenStreaming, setUseTokenStreaming] = useSessionState('tokenStreaming', defaultPresets.tokenStreaming);
5807+
const [promptPreviewChunks, setPromptPreviewChunks] = useState([]);
5808+
const [promptPreviewReroll, setPromptPreviewReroll] = useState(0);
5809+
const [showPromptPreview, setShowPromptPreview] = useSessionState('promptPreview', defaultPresets.promptPreview);
5810+
const [promptPreviewTokens, setPromptPreviewTokens] = useSessionState('promptPreviewTokens', defaultPresets.promptPreviewTokens);
57785811

57795812
function replacePlaceholders(string,placeholders) {
57805813
// give placeholders as json object
@@ -6176,6 +6209,31 @@
61766209
return replacePlaceholders(additionalContextPrompt, templateReplacements);
61776210
}, [additionalContextPrompt, templates, selectedTemplate]);
61786211

6212+
// predicts the prompt preview
6213+
useEffect(() => {
6214+
if (fimPromptInfo !== undefined || cancel || !showPromptPreview)
6215+
return;
6216+
6217+
setPromptPreviewChunks([]);
6218+
6219+
const ac = new AbortController();
6220+
const to = setTimeout(async () => {
6221+
const customParams = {
6222+
n_predict: promptPreviewTokens
6223+
};
6224+
6225+
const predicted = await predict(finalPromptText, promptChunks.length, (chunk) => {
6226+
setPromptPreviewChunks((c) => [...c, chunk]);
6227+
return true;
6228+
}, ac, customParams);
6229+
}, 500);
6230+
6231+
ac.signal.addEventListener('abort', () => clearTimeout(to));
6232+
return () => ac.abort();
6233+
}, [finalPromptText, showPromptPreview, promptPreviewReroll, promptPreviewTokens, cancel, endpoint, endpointAPI, endpointAPIKey]);
6234+
6235+
const promptPreviewText = useMemo(() => joinPrompt(promptPreviewChunks), [promptPreviewChunks]);
6236+
61796237
// predicts one {fill} placeholder
61806238
async function fillPredict() {
61816239
if (fimPromptInfo === undefined)
@@ -6270,8 +6328,8 @@
62706328
return messages;
62716329
}
62726330

6273-
async function predict(prompt = finalPromptText, chunkCount = promptChunks.length, callback = undefined) {
6274-
if (cancel) {
6331+
async function predict(prompt = finalPromptText, chunkCount = promptChunks.length, callback = undefined, abortController = undefined, customParams = {}) {
6332+
if (!abortController && cancel) {
62756333
cancel?.();
62766334

62776335
// llama.cpp server sometimes generates gibberish if we stop and
@@ -6287,17 +6345,32 @@
62876345
if (!callback && !restartedPredict && await fillPredict())
62886346
return true;
62896347

6290-
const ac = new AbortController();
6291-
const cancelThis = () => {
6292-
abortCompletion({
6293-
endpoint,
6294-
endpointAPI,
6295-
...(endpointAPI == API_AI_HORDE ? { hordeTaskId: hordeTaskId.current } : {}),
6296-
...(isMikupadEndpoint ? { proxyEndpoint: sessionStorage.proxyEndpoint } : {})
6297-
});
6298-
ac.abort();
6299-
};
6300-
setCancel(() => cancelThis);
6348+
let ac;
6349+
let cancelThis;
6350+
if (!abortController) {
6351+
ac = new AbortController();
6352+
cancelThis = () => {
6353+
abortCompletion({
6354+
endpoint,
6355+
endpointAPI,
6356+
...(endpointAPI == API_AI_HORDE ? { hordeTaskId: hordeTaskId.current } : {}),
6357+
...(isMikupadEndpoint ? { proxyEndpoint: sessionStorage.proxyEndpoint } : {})
6358+
});
6359+
ac.abort();
6360+
};
6361+
setCancel(() => cancelThis);
6362+
} else {
6363+
ac = abortController;
6364+
cancelThis = () => {
6365+
abortCompletion({
6366+
endpoint,
6367+
endpointAPI,
6368+
...(endpointAPI == API_AI_HORDE ? { hordeTaskId: hordeTaskId.current } : {}),
6369+
...(isMikupadEndpoint ? { proxyEndpoint: sessionStorage.proxyEndpoint } : {})
6370+
});
6371+
};
6372+
ac.signal.addEventListener('abort', cancelThis);
6373+
}
63016374
setLastError(undefined);
63026375

63036376
let predictCount = 0;
@@ -6425,7 +6498,8 @@
64256498
stream: useTokenStreaming,
64266499
...(JSON.parse(stoppingStrings).length ? { stop: JSON.parse(stoppingStrings) } : {}),
64276500
signal: ac.signal,
6428-
...(isMikupadEndpoint ? { proxyEndpoint: sessionStorage.proxyEndpoint } : {})
6501+
...(isMikupadEndpoint ? { proxyEndpoint: sessionStorage.proxyEndpoint } : {}),
6502+
...customParams
64296503
})) {
64306504
ac.signal.throwIfAborted();
64316505
if (chunk.stopping_word)
@@ -6478,6 +6552,8 @@
64786552
return false;
64796553
} finally {
64806554
setCancel(c => c === cancelThis ? null : c);
6555+
if (abortController)
6556+
ac.signal.removeEventListener('abort', cancelThis);
64816557
if (!callback) {
64826558
if (predictCount === 0)
64836559
undoStack.current.pop();
@@ -6681,7 +6757,32 @@
66816757
}, [promptText]);
66826758

66836759
useLayoutEffect(() => {
6684-
if (cancel)
6760+
const elem = promptArea.current;
6761+
const previewElem = promptPreviewElement.current;
6762+
if (!elem || !previewElem)
6763+
return;
6764+
const oldHeight = elem.scrollHeight;
6765+
const atBottom = (elem.scrollTarget ?? elem.scrollTop) + elem.clientHeight + 1 > oldHeight;
6766+
previewElem.textContent = promptPreviewText;
6767+
elem.style.paddingBottom = previewElem.offsetHeight + 'px';
6768+
requestAnimationFrame(() => {
6769+
const newHeight = elem.scrollHeight;
6770+
if (atBottom && oldHeight !== newHeight) {
6771+
if (elem.scrollHeight - (elem.scrollTop + elem.clientHeight + 1) >= 100) {
6772+
// smooth scroll isn't keeping up with prediction speed =(
6773+
useScrollSmoothing.current = false;
6774+
}
6775+
elem.scrollTarget = newHeight - elem.clientHeight;
6776+
elem.scrollTo({
6777+
top: newHeight - elem.clientHeight,
6778+
behavior: useScrollSmoothing.current ? 'smooth' : 'instant',
6779+
});
6780+
}
6781+
});
6782+
}, [promptPreviewText]);
6783+
6784+
useLayoutEffect(() => {
6785+
if (cancel || promptPreviewText)
66856786
return;
66866787
promptArea.current.scrollTarget = undefined;
66876788
promptArea.current.scrollTop = savedScrollTop;
@@ -6755,7 +6856,22 @@
67556856
predict();
67566857
break;
67576858
case 'false:false:false:Escape':
6758-
cancel();
6859+
if (cancel) {
6860+
cancel();
6861+
} else if (promptPreviewText.length !== 0) {
6862+
setPromptPreviewReroll((r) => r + 1);
6863+
}
6864+
break;
6865+
case 'false:false:false:Tab':
6866+
if (promptPreviewText.length === 0)
6867+
break;
6868+
6869+
setPromptChunks(p => [
6870+
...p,
6871+
...promptPreviewChunks
6872+
]);
6873+
setTokens(t => t + promptPreviewChunks.length);
6874+
setPromptPreviewChunks([]);
67596875
break;
67606876
case 'false:true:false:r':
67616877
case 'false:false:true:r':
@@ -7090,6 +7206,7 @@
70907206
redoStack.current = [];
70917207
undoStack.current = [];
70927208
setUndoHovered(false);
7209+
setPromptPreviewChunks([]);
70937210
setTitleToSession();
70947211
}
70957212
function onSessionError() {
@@ -7194,8 +7311,13 @@
71947311
id="prompt-area"
71957312
onInput=${onInput}
71967313
onScroll=${onScroll}
7197-
onContextMenu=${onContextMenu}/>
7198-
<div ref=${promptOverlay} id="prompt-overlay" aria-hidden>
7314+
onContextMenu=${onContextMenu}
7315+
...${showPromptPreview && { style: { 'padding-bottom': promptPreviewElement.current?.offsetHeight ?? '0px' } }}/>
7316+
<div
7317+
ref=${promptOverlay}
7318+
id="prompt-overlay"
7319+
aria-hidden
7320+
...${showPromptPreview && { style: { 'padding-bottom': promptPreviewElement.current?.offsetHeight ?? '0px' } }}>
71997321
${tokenHighlightMode !== -1 ? html`
72007322
${promptChunks.map((chunk, i) => {
72017323
const getRatioColor = (ratio) => {
@@ -7228,9 +7350,12 @@
72287350
data-promptchunk=${i}
72297351
style=${bgColor ? { '--bg-color': bgColor } : {}}
72307352
className=${`${(tokenHighlightMode === 1 && !isCurrent) || chunk.type === 'user' ? 'user' : 'machine'} ${isCurrent ? 'current' : ''} ${isNextUndo ? 'erase' : ''}`}>
7231-
${(chunk.content === '\n' ? ' \n' : chunk.content) + (i === promptChunks.length - 1 && chunk.content.endsWith('\n') ? '\u00a0' : '')}
7353+
${(chunk.content === '\n' ? ' \n' : chunk.content) + (i === promptChunks.length - 1 && chunk.content.endsWith('\n') && promptPreviewText?.length === 0 ? '\u00a0' : '')}
72327354
</span>`;
72337355
})}` : null}
7356+
${(showPromptPreview && promptPreviewText?.length) ? html`
7357+
<span ref=${promptPreviewElement} className="preview"></span>
7358+
<span class="preview nudge">Tab</span>` : null}
72347359
</div>
72357360
<${SearchAndReplaceWidget}
72367361
isOpen=${modalState.searchAndReplace}
@@ -7320,7 +7445,12 @@
73207445
title="If enabled, the chat API endpoint will be used, and the prompt will be split into chat messages based on the delimiters defined in the selected instruct template."
73217446
disabled=${!!cancel} value=${useChatAPI} onValueChange=${setUseChatAPI}/>`}
73227447
<${Checkbox} label="Token Streaming"
7323-
disabled=${!!cancel} value=${useTokenStreaming} onValueChange=${setUseTokenStreaming}/>`}
7448+
disabled=${!!cancel} value=${useTokenStreaming} onValueChange=${setUseTokenStreaming}/>
7449+
<${Checkbox} label="Prediction Preview"
7450+
disabled=${!!cancel} value=${showPromptPreview} onValueChange=${setShowPromptPreview}/>
7451+
${showPromptPreview && html`
7452+
<${InputBox} label="Max Preview Tokens" type="text" inputmode="numeric"
7453+
readOnly=${!!cancel} value=${promptPreviewTokens} onValueChange=${setPromptPreviewTokens}/>`}`}
73247454
<div className="buttons instructTemplateSidebar">
73257455
<${SelectBoxTemplate}
73267456
label="Instruct Template"

0 commit comments

Comments
 (0)