perf: fetch only new inputs on refresh of inputs history

backend/windmill-api/openapi.yaml

@@ -10159,6 +10159,10 @@ paths:
           in: query
           schema:
             type: boolean
+        - name: since
+          in: query
+          schema:
+            type: string
       responses:
         "200":
           description: Input history for completed jobs

backend/windmill-api/src/inputs.rs

@@ -18,11 +18,12 @@ use serde_json::Value;
 use sqlx::{types::Uuid, FromRow};
 use std::{
     fmt::{Display, Formatter},
+    str::FromStr,
     vec,
 };
 use windmill_common::{
     db::UserDB,
-    error::JsonResult,
+    error::{self, JsonResult},
     jobs::JobKind,
     scripts::to_i64,
     utils::{not_found_if_none, paginate, Pagination},
@@ -118,6 +119,7 @@ pub struct CompletedJobMini {
 #[derive(Deserialize)]
 struct GetInputHistory {
     include_preview: Option<bool>,
+    since: Option<String>,
 }
 
 async fn get_input_history(
@@ -135,7 +137,8 @@ async fn get_input_history(
     let sql = &format!(
         "select id, v2_job.created_at, created_by, 'null'::jsonb as args, status = 'success' as success from v2_job JOIN v2_job_completed USING (id) \
         where {} = $1 and kind = any($2) and v2_job.workspace_id = $3 AND v2_job_completed.status != 'skipped' \
-        order by v2_job.created_at desc limit $4 offset $5",
+        AND v2_job_completed.started_at >= $4 \
+        order by v2_job.created_at desc limit $5 offset $6",
         r.runnable_type.column_name()
     );
 
@@ -156,9 +159,16 @@ async fn get_input_history(
         kind => vec![kind],
     };
 
+    let since: DateTime<chrono::Local> = if let Some(date) = g.since {
+        DateTime::from_str(date.as_str()).map_err(|e| error::Error::BadRequest(format!("{e}")))?
+    } else {
+        DateTime::default()
+    };
+
     let rows = query
         .bind(job_kinds)
         .bind(&w_id)
+        .bind(since)
         .bind(per_page as i32)
         .bind(offset as i32)
         .fetch_all(&mut *tx)

frontend/src/lib/components/HistoricList.svelte

@@ -10,30 +10,40 @@
 	export let selected: string | undefined = undefined
 
 	let infiniteList: InfiniteList | undefined = undefined
-	let loadInputsPageFn: ((page: number, perPage: number) => Promise<any>) | undefined = undefined
+	let loadInputsPageFn:
+		| ((page: number, perPage: number, discovery: boolean) => Promise<any>)
+		| undefined = undefined
 
 	export function refresh() {
 		if (infiniteList) {
 			infiniteList.loadData('refresh')
 		}
 	}
 	let cachedArgs: Record<string, any> = {}
+	let items: any[] = []
+	let potentialItems: any[] = []
+	let perPageBind = 0
+	let lastChecked: string | undefined = undefined
 
 	let interval: NodeJS.Timeout | undefined = undefined
 	function initLoadInputs() {
 		interval && clearInterval(interval)
 		interval = setInterval(() => {
 			refresh()
 		}, 10000)
-		loadInputsPageFn = async (page: number, perPage: number) => {
-			const inputs = await InputService.getInputHistory({
+		loadInputsPageFn = async (page: number, perPage: number, discovery: boolean) => {
+			const request = InputService.getInputHistory({
 				workspace: $workspaceStore!,
 				runnableId,
 				runnableType,
 				page,
 				perPage,
-				includePreview: true
+				includePreview: true,
+				// If it is discovery, then we would like to fetch all values
+				since: !discovery ? lastChecked : undefined
 			})
+			if (!discovery) lastChecked = new Date().toJSON()
+			const inputs = await request
 
 			const inputsWithPayload = await Promise.all(
 				inputs.map(async (input) => {
@@ -58,7 +68,22 @@
 					}
 				})
 			)
-			return inputsWithPayload
+			if (!discovery) {
+				// Add new items to beginning
+				items.unshift(...inputsWithPayload)
+				// We need to know when to apply potential items,
+				// it only happens when InfiniteList decides to expand list
+				//
+				// We cannot apply potential items right after fetch,
+				// because that would trigger expansion in list
+				// and old items will be loading by 10 every reload
+				if (perPageBind != perPage) items.push(...potentialItems), (perPageBind = perPage)
+				return items
+			} else {
+				// Save discovered items to buffer and apply later
+				potentialItems = inputsWithPayload
+				return potentialItems
+			}
 		}
 		infiniteList?.setLoader(loadInputsPageFn)
 	}

frontend/src/lib/components/InfiniteList.svelte

@@ -18,7 +18,7 @@
 	let hasAlreadyFailed = false
 	let hovered: any | undefined = undefined
 	let initLoad = false
-	let loadInputs: ((page: number, perPage: number) => Promise<any[]>) | undefined = undefined
+	let loadInputs: ((page: number, perPage: number, discovery: boolean) => Promise<any[]>) | undefined = undefined
 	let deleteItemFn: ((id: any) => Promise<any>) | undefined = undefined
 
 	export async function loadData(loadOption: 'refresh' | 'forceRefresh' | 'loadMore' = 'loadMore') {
@@ -33,7 +33,7 @@
 		}
 
 		try {
-			const newItems = await loadInputs(1, page * perPage)
+			const newItems = await loadInputs(1, page * perPage, false)
 
 			if (
 				loadOption === 'refresh' &&
@@ -63,7 +63,7 @@
 			page = Math.ceil(items.length / perPage)
 			hasMore = items.length === perPage * page
 			if (hasMore) {
-				const potentialNewItems = await loadInputs(page + 1, perPage)
+				const potentialNewItems = await loadInputs(page + 1, perPage, true)
 				hasMore = potentialNewItems.length > 0
 			}
 			initLoad = true
@@ -96,7 +96,7 @@
 		}
 	}
 
-	export async function setLoader(loader: (page: number, perPage: number) => Promise<any[]>) {
+	export async function setLoader(loader: (page: number, perPage: number, discovery: boolean) => Promise<any[]>) {
 		loadInputs = loader
 		loadData('forceRefresh')
 	}

frontend/src/lib/components/SavedInputsPicker.svelte

@@ -59,7 +59,7 @@
 	}
 
 	function initLoadInputs() {
-		const loadInputsPageFn = async (page: number, perPage: number) => {
+		const loadInputsPageFn = async (page: number, perPage: number, _discovery: boolean) => {
 			const inputs = await InputService.listInputs({
 				workspace: $workspaceStore!,
 				runnableId,

frontend/src/lib/components/triggers/CaptureTable.svelte

@@ -67,7 +67,7 @@
 	}
 
 	function initLoadCaptures(testKind: 'preprocessor' | 'main' = 'main') {
-		const loadInputsPageFn = async (page: number, perPage: number) => {
+		const loadInputsPageFn = async (page: number, perPage: number, _discovery: boolean) => {
 			const captures = await CaptureService.listCaptures({
 				workspace: $workspaceStore!,
 				runnableKind: isFlow ? 'flow' : 'script',