fix: improve que job indices for faster performances

windmill-labs · Feb 17, 2025 · 5cdc1b8 · 5cdc1b8
1 parent fe33729
commit 5cdc1b8
Show file tree

Hide file tree

Showing 7 changed files with 79 additions and 14 deletions.
diff --git a/backend/.sqlx/query-0efb16cbf130ec6e9922ecc82a95b252449bd569df374e40ce8820fc3d75a0f0.json b/backend/.sqlx/query-0efb16cbf130ec6e9922ecc82a95b252449bd569df374e40ce8820fc3d75a0f0.json
diff --git a/backend/.sqlx/query-3738096c29ab9d964be8a74bfd14ff1d599049ebefdaf97a017c9cef8d52ce20.json b/backend/.sqlx/query-3738096c29ab9d964be8a74bfd14ff1d599049ebefdaf97a017c9cef8d52ce20.json
diff --git a/backend/.sqlx/query-3bbde0fa35d935ec2dd8bd1fb14cfecf48305f5f4b644b3c35355074e1ccce28.json b/backend/.sqlx/query-3bbde0fa35d935ec2dd8bd1fb14cfecf48305f5f4b644b3c35355074e1ccce28.json
diff --git a/backend/.sqlx/query-8263fe28097e094cbdbdcd16668d9347206d2deaa99f9a4541101e610f84a50a.json b/backend/.sqlx/query-8263fe28097e094cbdbdcd16668d9347206d2deaa99f9a4541101e610f84a50a.json
diff --git a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json
diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs
@@ -655,6 +655,23 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> {
             .await?;
     });
 
+    run_windmill_migration!("v2_improve_v2_queued_jobs_indices", &db, |tx| {
+        sqlx::query!("CREATE INDEX CONCURRENTLY queue_sort_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) WHERE running = false")
+            .execute(db)
+            .await?;
+
+        sqlx::query!("CREATE INDEX CONCURRENTLY queue_sort_2_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for) WHERE running = false")
+            .execute(db)
+            .await?;
+
+        sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS queue_sort")
+            .execute(db)
+            .await?;
+
+        sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS queue_sort_2")
+            .execute(db)
+            .await?;
+    });
     Ok(())
 }
 

diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs
@@ -281,26 +281,26 @@ pub async fn connect_db(
 pub async fn connect(
     database_url: &str,
     max_connections: u32,
-    worker_mode: bool,
+    _worker_mode: bool,
 ) -> Result<sqlx::Pool<sqlx::Postgres>, error::Error> {
     use std::time::Duration;
 
     sqlx::postgres::PgPoolOptions::new()
         .min_connections((max_connections / 5).clamp(3, max_connections))
         .max_connections(max_connections)
         .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
-        .after_connect(move |conn, _| {
-            if worker_mode {
-                Box::pin(async move {
-                    sqlx::query("SET enable_seqscan = OFF;")
-                        .execute(conn)
-                        .await?;
-                    Ok(())
-                })
-            } else {
-                Box::pin(async move { Ok(()) })
-            }
-        })
+        // .after_connect(move |conn, _| {
+        //     if worker_mode {
+        //         Box::pin(async move {
+        //             sqlx::query("SET enable_seqscan = OFF;")
+        //                 .execute(conn)
+        //                 .await?;
+        //             Ok(())
+        //         })
+        //     } else {
+        //         Box::pin(async move { Ok(()) })
+        //     }
+        // })
         .connect_with(
             sqlx::postgres::PgConnectOptions::from_str(database_url)?.statement_cache_capacity(400),
         )