From dfcfcda7b929c1e2620aaed774d820b8bede5b15 Mon Sep 17 00:00:00 2001 From: pyranota Date: Mon, 17 Feb 2025 23:49:03 +0300 Subject: [PATCH 1/6] perf: fetch only new inputs on refresh of inputs history --- backend/windmill-api/openapi.yaml | 4 +++ backend/windmill-api/src/inputs.rs | 14 ++++++-- .../src/lib/components/HistoricList.svelte | 35 ++++++++++++++++--- .../src/lib/components/InfiniteList.svelte | 8 ++--- 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 87af4c763d63c..4fffcfd6963ac 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -10159,6 +10159,10 @@ paths: in: query schema: type: boolean + - name: since + in: query + schema: + type: string responses: "200": description: Input history for completed jobs diff --git a/backend/windmill-api/src/inputs.rs b/backend/windmill-api/src/inputs.rs index ec200b44640d3..2345f6b3b6a39 100644 --- a/backend/windmill-api/src/inputs.rs +++ b/backend/windmill-api/src/inputs.rs @@ -18,11 +18,12 @@ use serde_json::Value; use sqlx::{types::Uuid, FromRow}; use std::{ fmt::{Display, Formatter}, + str::FromStr, vec, }; use windmill_common::{ db::UserDB, - error::JsonResult, + error::{self, JsonResult}, jobs::JobKind, scripts::to_i64, utils::{not_found_if_none, paginate, Pagination}, @@ -118,6 +119,7 @@ pub struct CompletedJobMini { #[derive(Deserialize)] struct GetInputHistory { include_preview: Option, + since: Option, } async fn get_input_history( @@ -135,7 +137,8 @@ async fn get_input_history( let sql = &format!( "select id, v2_job.created_at, created_by, 'null'::jsonb as args, status = 'success' as success from v2_job JOIN v2_job_completed USING (id) \ where {} = $1 and kind = any($2) and v2_job.workspace_id = $3 AND v2_job_completed.status != 'skipped' \ - order by v2_job.created_at desc limit $4 offset $5", + AND v2_job_completed.started_at >= $4 \ + order by v2_job.created_at desc limit $5 offset $6", r.runnable_type.column_name() ); @@ -156,9 +159,16 @@ async fn get_input_history( kind => vec![kind], }; + let since: DateTime = if let Some(date) = g.since { + DateTime::from_str(date.as_str()).map_err(|e| error::Error::BadRequest(format!("{e}")))? + } else { + DateTime::default() + }; + let rows = query .bind(job_kinds) .bind(&w_id) + .bind(since) .bind(per_page as i32) .bind(offset as i32) .fetch_all(&mut *tx) diff --git a/frontend/src/lib/components/HistoricList.svelte b/frontend/src/lib/components/HistoricList.svelte index c3118d39004f3..b5fa44a539cf6 100644 --- a/frontend/src/lib/components/HistoricList.svelte +++ b/frontend/src/lib/components/HistoricList.svelte @@ -10,7 +10,9 @@ export let selected: string | undefined = undefined let infiniteList: InfiniteList | undefined = undefined - let loadInputsPageFn: ((page: number, perPage: number) => Promise) | undefined = undefined + let loadInputsPageFn: + | ((page: number, perPage: number, discovery: boolean) => Promise) + | undefined = undefined export function refresh() { if (infiniteList) { @@ -18,6 +20,10 @@ } } let cachedArgs: Record = {} + let items: any[] = [] + let potentialItems: any[] = [] + let perPageBind = 0 + let lastChecked: string | undefined = undefined let interval: NodeJS.Timeout | undefined = undefined function initLoadInputs() { @@ -25,15 +31,19 @@ interval = setInterval(() => { refresh() }, 10000) - loadInputsPageFn = async (page: number, perPage: number) => { - const inputs = await InputService.getInputHistory({ + loadInputsPageFn = async (page: number, perPage: number, discovery: boolean) => { + const request = InputService.getInputHistory({ workspace: $workspaceStore!, runnableId, runnableType, page, perPage, - includePreview: true + includePreview: true, + // If it is discovery, then we would like to fetch all values + since: !discovery ? lastChecked : undefined }) + if (!discovery) lastChecked = new Date().toJSON() + const inputs = await request const inputsWithPayload = await Promise.all( inputs.map(async (input) => { @@ -58,7 +68,22 @@ } }) ) - return inputsWithPayload + if (!discovery) { + // Add new items to beginning + items.unshift(...inputsWithPayload) + // We need to know when to apply potential items, + // it only happens when InfiniteList decides to expand list + // + // We cannot apply potential items right after fetch, + // because that would trigger expansion in list + // and old items will be loading by 10 every reload + if (perPageBind != perPage) items.push(...potentialItems), (perPageBind = perPage) + return items + } else { + // Save discovered items to buffer and apply later + potentialItems = inputsWithPayload + return potentialItems + } } infiniteList?.setLoader(loadInputsPageFn) } diff --git a/frontend/src/lib/components/InfiniteList.svelte b/frontend/src/lib/components/InfiniteList.svelte index e8caaefdb7692..9bf425017c374 100644 --- a/frontend/src/lib/components/InfiniteList.svelte +++ b/frontend/src/lib/components/InfiniteList.svelte @@ -18,7 +18,7 @@ let hasAlreadyFailed = false let hovered: any | undefined = undefined let initLoad = false - let loadInputs: ((page: number, perPage: number) => Promise) | undefined = undefined + let loadInputs: ((page: number, perPage: number, discovery: boolean) => Promise) | undefined = undefined let deleteItemFn: ((id: any) => Promise) | undefined = undefined export async function loadData(loadOption: 'refresh' | 'forceRefresh' | 'loadMore' = 'loadMore') { @@ -33,7 +33,7 @@ } try { - const newItems = await loadInputs(1, page * perPage) + const newItems = await loadInputs(1, page * perPage, false) if ( loadOption === 'refresh' && @@ -63,7 +63,7 @@ page = Math.ceil(items.length / perPage) hasMore = items.length === perPage * page if (hasMore) { - const potentialNewItems = await loadInputs(page + 1, perPage) + const potentialNewItems = await loadInputs(page + 1, perPage, true) hasMore = potentialNewItems.length > 0 } initLoad = true @@ -96,7 +96,7 @@ } } - export async function setLoader(loader: (page: number, perPage: number) => Promise) { + export async function setLoader(loader: (page: number, perPage: number, discovery: boolean) => Promise) { loadInputs = loader loadData('forceRefresh') } From 3616df19403dceca9cc25409adbacdaa875c7e5d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 17 Feb 2025 21:54:00 +0100 Subject: [PATCH 2/6] nit benchmarks improvement --- benchmarks/benchmark_oneoff.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index 62a3e3d5f7b02..2800eadebb640 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -26,7 +26,7 @@ async function verifyOutputs(uuids: string[], workspace: string) { incorrectResults++; } if (job.result !== uuid) { - console.log(`Job ${uuid} did not output the correct value`); + console.log(`Job ${uuid} did not output the correct value: ${JSON.stringify(job.result)}`); incorrectResults++; } } catch (_) { From 50b9b27b9ce597eedc12321f808fdd9ccffa7365 Mon Sep 17 00:00:00 2001 From: pyranota Date: Tue, 18 Feb 2025 00:08:55 +0300 Subject: [PATCH 3/6] add arguments to signatures to be extra safe --- frontend/src/lib/components/SavedInputsPicker.svelte | 2 +- frontend/src/lib/components/triggers/CaptureTable.svelte | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/components/SavedInputsPicker.svelte b/frontend/src/lib/components/SavedInputsPicker.svelte index 5eaf67e3aa9c0..6535889eab21f 100644 --- a/frontend/src/lib/components/SavedInputsPicker.svelte +++ b/frontend/src/lib/components/SavedInputsPicker.svelte @@ -59,7 +59,7 @@ } function initLoadInputs() { - const loadInputsPageFn = async (page: number, perPage: number) => { + const loadInputsPageFn = async (page: number, perPage: number, _discovery: boolean) => { const inputs = await InputService.listInputs({ workspace: $workspaceStore!, runnableId, diff --git a/frontend/src/lib/components/triggers/CaptureTable.svelte b/frontend/src/lib/components/triggers/CaptureTable.svelte index 70596003fc26e..c4c68f516dee1 100644 --- a/frontend/src/lib/components/triggers/CaptureTable.svelte +++ b/frontend/src/lib/components/triggers/CaptureTable.svelte @@ -67,7 +67,7 @@ } function initLoadCaptures(testKind: 'preprocessor' | 'main' = 'main') { - const loadInputsPageFn = async (page: number, perPage: number) => { + const loadInputsPageFn = async (page: number, perPage: number, _discovery: boolean) => { const captures = await CaptureService.listCaptures({ workspace: $workspaceStore!, runnableKind: isFlow ? 'flow' : 'script', From 27ca01bda72cc97c56cae29abf07e8cdefbfeda6 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Tue, 18 Feb 2025 00:20:44 +0300 Subject: [PATCH 4/6] Update frontend/src/lib/components/HistoricList.svelte Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> --- frontend/src/lib/components/HistoricList.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/HistoricList.svelte b/frontend/src/lib/components/HistoricList.svelte index b5fa44a539cf6..cc84852030ebd 100644 --- a/frontend/src/lib/components/HistoricList.svelte +++ b/frontend/src/lib/components/HistoricList.svelte @@ -77,7 +77,7 @@ // We cannot apply potential items right after fetch, // because that would trigger expansion in list // and old items will be loading by 10 every reload - if (perPageBind != perPage) items.push(...potentialItems), (perPageBind = perPage) + if (perPageBind != perPage) { items.push(...potentialItems); perPageBind = perPage; } return items } else { // Save discovered items to buffer and apply later From 45df9228e0cefd5ec6e63dd98eb61524a1887f34 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Tue, 18 Feb 2025 15:34:28 +0300 Subject: [PATCH 5/6] Update frontend/src/lib/components/SavedInputsPicker.svelte Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> --- frontend/src/lib/components/SavedInputsPicker.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/SavedInputsPicker.svelte b/frontend/src/lib/components/SavedInputsPicker.svelte index 6535889eab21f..5eaf67e3aa9c0 100644 --- a/frontend/src/lib/components/SavedInputsPicker.svelte +++ b/frontend/src/lib/components/SavedInputsPicker.svelte @@ -59,7 +59,7 @@ } function initLoadInputs() { - const loadInputsPageFn = async (page: number, perPage: number, _discovery: boolean) => { + const loadInputsPageFn = async (page: number, perPage: number) => { const inputs = await InputService.listInputs({ workspace: $workspaceStore!, runnableId, From d5a3ead1ffffed0a705d732c51e6736ad2aaa9e6 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Tue, 18 Feb 2025 15:34:52 +0300 Subject: [PATCH 6/6] Update frontend/src/lib/components/triggers/CaptureTable.svelte Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> --- frontend/src/lib/components/triggers/CaptureTable.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/triggers/CaptureTable.svelte b/frontend/src/lib/components/triggers/CaptureTable.svelte index c4c68f516dee1..70596003fc26e 100644 --- a/frontend/src/lib/components/triggers/CaptureTable.svelte +++ b/frontend/src/lib/components/triggers/CaptureTable.svelte @@ -67,7 +67,7 @@ } function initLoadCaptures(testKind: 'preprocessor' | 'main' = 'main') { - const loadInputsPageFn = async (page: number, perPage: number, _discovery: boolean) => { + const loadInputsPageFn = async (page: number, perPage: number) => { const captures = await CaptureService.listCaptures({ workspace: $workspaceStore!, runnableKind: isFlow ? 'flow' : 'script',