From 73630a9cf8ad2de091e8b7e919c891ea1ceabc45 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Mon, 24 Feb 2025 20:18:00 +0100 Subject: [PATCH 1/8] feat: track workspace runnables used in flows --- ...52a6591e7593b1a7c3d7a6848587679a48187.json | 15 ++ ...7ba20fe2ae7990db7bec0d44b9f2a9df3792b.json | 17 +++ ...3242b54d77b46b5f785ae66b8104f89f43219.json | 24 ++++ ...18cd141e68cf83a7dcae7d3c1b8a7ba94425b.json | 15 ++ ...4f0bf040da3af9e2eac3229264ec95b4755d8.json | 16 +++ ...a4420f232867a23cacb9a62b386f129a86a4b.json | 16 +++ ...05131521_flow_workspace_runnables.down.sql | 1 + ...0205131521_flow_workspace_runnables.up.sql | 9 ++ backend/windmill-api/src/capture.rs | 13 +- backend/windmill-api/src/flows.rs | 128 +++++++++++++++++- backend/windmill-api/src/users.rs | 16 +++ backend/windmill-api/src/utils.rs | 7 + backend/windmill-api/src/workspaces_extra.rs | 8 ++ 13 files changed, 274 insertions(+), 11 deletions(-) create mode 100644 backend/.sqlx/query-2bb2cf6accb18d3e37a63388cca52a6591e7593b1a7c3d7a6848587679a48187.json create mode 100644 backend/.sqlx/query-62986d32e7c0426bbfc2238e1fc7ba20fe2ae7990db7bec0d44b9f2a9df3792b.json create mode 100644 backend/.sqlx/query-9aeee333b1dbe58ba819ba3b2713242b54d77b46b5f785ae66b8104f89f43219.json create mode 100644 backend/.sqlx/query-acbea8740b28c26942c50edcf5618cd141e68cf83a7dcae7d3c1b8a7ba94425b.json create mode 100644 backend/.sqlx/query-b656927cd70b6667f3c72186ec04f0bf040da3af9e2eac3229264ec95b4755d8.json create mode 100644 backend/.sqlx/query-dc58e5b4715601a93b3c01a2564a4420f232867a23cacb9a62b386f129a86a4b.json create mode 100644 backend/migrations/20250205131521_flow_workspace_runnables.down.sql create mode 100644 backend/migrations/20250205131521_flow_workspace_runnables.up.sql diff --git a/backend/.sqlx/query-2bb2cf6accb18d3e37a63388cca52a6591e7593b1a7c3d7a6848587679a48187.json b/backend/.sqlx/query-2bb2cf6accb18d3e37a63388cca52a6591e7593b1a7c3d7a6848587679a48187.json new file mode 100644 index 0000000000000..11e7df65bfa29 --- /dev/null +++ b/backend/.sqlx/query-2bb2cf6accb18d3e37a63388cca52a6591e7593b1a7c3d7a6848587679a48187.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE flow_workspace_runnables SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2bb2cf6accb18d3e37a63388cca52a6591e7593b1a7c3d7a6848587679a48187" +} diff --git a/backend/.sqlx/query-62986d32e7c0426bbfc2238e1fc7ba20fe2ae7990db7bec0d44b9f2a9df3792b.json b/backend/.sqlx/query-62986d32e7c0426bbfc2238e1fc7ba20fe2ae7990db7bec0d44b9f2a9df3792b.json new file mode 100644 index 0000000000000..5d0d7126756aa --- /dev/null +++ b/backend/.sqlx/query-62986d32e7c0426bbfc2238e1fc7ba20fe2ae7990db7bec0d44b9f2a9df3792b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "62986d32e7c0426bbfc2238e1fc7ba20fe2ae7990db7bec0d44b9f2a9df3792b" +} diff --git a/backend/.sqlx/query-9aeee333b1dbe58ba819ba3b2713242b54d77b46b5f785ae66b8104f89f43219.json b/backend/.sqlx/query-9aeee333b1dbe58ba819ba3b2713242b54d77b46b5f785ae66b8104f89f43219.json new file mode 100644 index 0000000000000..271a508a830fa --- /dev/null +++ b/backend/.sqlx/query-9aeee333b1dbe58ba819ba3b2713242b54d77b46b5f785ae66b8104f89f43219.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT f.path\n FROM flow_workspace_runnables fwr \n JOIN flow f \n ON fwr.flow_path = f.path AND fwr.workspace_id = f.workspace_id\n WHERE fwr.runnable_path = $1 AND fwr.runnable_is_flow = $2 AND fwr.workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "9aeee333b1dbe58ba819ba3b2713242b54d77b46b5f785ae66b8104f89f43219" +} diff --git a/backend/.sqlx/query-acbea8740b28c26942c50edcf5618cd141e68cf83a7dcae7d3c1b8a7ba94425b.json b/backend/.sqlx/query-acbea8740b28c26942c50edcf5618cd141e68cf83a7dcae7d3c1b8a7ba94425b.json new file mode 100644 index 0000000000000..410aed9cc5d8f --- /dev/null +++ b/backend/.sqlx/query-acbea8740b28c26942c50edcf5618cd141e68cf83a7dcae7d3c1b8a7ba94425b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_workspace_runnables WHERE flow_path = $1 AND workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "acbea8740b28c26942c50edcf5618cd141e68cf83a7dcae7d3c1b8a7ba94425b" +} diff --git a/backend/.sqlx/query-b656927cd70b6667f3c72186ec04f0bf040da3af9e2eac3229264ec95b4755d8.json b/backend/.sqlx/query-b656927cd70b6667f3c72186ec04f0bf040da3af9e2eac3229264ec95b4755d8.json new file mode 100644 index 0000000000000..c05edddc9ffd6 --- /dev/null +++ b/backend/.sqlx/query-b656927cd70b6667f3c72186ec04f0bf040da3af9e2eac3229264ec95b4755d8.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE flow_workspace_runnables SET flow_path = REGEXP_REPLACE(flow_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE flow_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b656927cd70b6667f3c72186ec04f0bf040da3af9e2eac3229264ec95b4755d8" +} diff --git a/backend/.sqlx/query-dc58e5b4715601a93b3c01a2564a4420f232867a23cacb9a62b386f129a86a4b.json b/backend/.sqlx/query-dc58e5b4715601a93b3c01a2564a4420f232867a23cacb9a62b386f129a86a4b.json new file mode 100644 index 0000000000000..d3b8f33ff0e52 --- /dev/null +++ b/backend/.sqlx/query-dc58e5b4715601a93b3c01a2564a4420f232867a23cacb9a62b386f129a86a4b.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE flow_workspace_runnables SET runnable_path = REGEXP_REPLACE(runnable_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE runnable_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "dc58e5b4715601a93b3c01a2564a4420f232867a23cacb9a62b386f129a86a4b" +} diff --git a/backend/migrations/20250205131521_flow_workspace_runnables.down.sql b/backend/migrations/20250205131521_flow_workspace_runnables.down.sql new file mode 100644 index 0000000000000..930245a2363c7 --- /dev/null +++ b/backend/migrations/20250205131521_flow_workspace_runnables.down.sql @@ -0,0 +1 @@ +DROP TABLE flow_workspace_runnables; \ No newline at end of file diff --git a/backend/migrations/20250205131521_flow_workspace_runnables.up.sql b/backend/migrations/20250205131521_flow_workspace_runnables.up.sql new file mode 100644 index 0000000000000..c7f6c35e505d8 --- /dev/null +++ b/backend/migrations/20250205131521_flow_workspace_runnables.up.sql @@ -0,0 +1,9 @@ +CREATE TABLE flow_workspace_runnables ( + flow_path VARCHAR(255) NOT NULL, + runnable_path VARCHAR(255) NOT NULL, + runnable_is_flow BOOLEAN NOT NULL, + workspace_id VARCHAR(50) NOT NULL, + FOREIGN KEY (workspace_id, flow_path) REFERENCES flow (workspace_id, path) ON DELETE CASCADE +); + +CREATE INDEX flow_workspace_runnable_path_is_flow_idx ON flow_workspace_runnables (runnable_path, runnable_is_flow, workspace_id); diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 7606822d656ac..5ada4204ad105 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -52,6 +52,7 @@ use crate::{ args::WebhookArgs, db::{ApiAuthed, DB}, users::fetch_api_authed, + utils::RunnableKind, }; const KEEP_LAST: i64 = 20; @@ -139,7 +140,7 @@ pub struct KafkaTriggerConfig { pub struct SqsTriggerConfig { pub queue_url: String, pub aws_resource_path: String, - pub message_attributes: Option> + pub message_attributes: Option>, } #[cfg(all(feature = "enterprise", feature = "nats"))] @@ -300,8 +301,7 @@ async fn set_config( #[cfg(feature = "postgres_trigger")] let nc = if let TriggerKind::Postgres = nc.trigger_kind { set_postgres_trigger_config(&w_id, authed.clone(), &db, user_db.clone(), nc).await? - } - else { + } else { nc }; @@ -362,13 +362,6 @@ struct Capture { trigger_extra: Option>>, } -#[derive(Deserialize)] -#[serde(rename_all = "lowercase")] -enum RunnableKind { - Script, - Flow, -} - #[derive(Deserialize)] struct ListCapturesQuery { trigger_kind: Option, diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 57770ff674ee5..fb5c07fb86c7a 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -12,7 +12,7 @@ use crate::db::ApiAuthed; use crate::triggers::{ get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail, }; -use crate::utils::WithStarredInfoQuery; +use crate::utils::{RunnableKind, WithStarredInfoQuery}; use crate::{ db::DB, schedule::clear_schedule, @@ -33,6 +33,7 @@ use sql_builder::prelude::*; use sqlx::{FromRow, Postgres, Transaction}; use windmill_audit::audit_ee::audit_log; use windmill_audit::ActionKind; +use windmill_common::flows::{FlowModule, FlowModuleValue}; use windmill_common::utils::query_elems_from_hub; use windmill_common::worker::to_raw_value; use windmill_common::HUB_BASE_URL; @@ -65,6 +66,10 @@ pub fn workspaced_service() -> Router { .route("/list_paths", get(list_paths)) .route("/history/p/*path", get(get_flow_history)) .route("/get_latest_version/*path", get(get_latest_version)) + .route( + "/list_paths_from_workspace_runnable/:runnable_kind/*path", + get(list_paths_from_workspace_runnable), + ) .route( "/history_update/v/:version/p/*path", post(update_flow_history), @@ -324,6 +329,115 @@ async fn check_path_conflict<'c>( return Ok(()); } +#[derive(Debug, Deserialize, Serialize)] +struct FlowWorkspaceRunnable { + runnable_path: String, + runnable_is_flow: bool, +} + +fn get_flow_workspace_runnables_from_modules( + modules: Vec, +) -> Result> { + let mut result = Vec::new(); + for m in modules { + match m.get_value()? { + FlowModuleValue::Script { path, .. } => { + result.push(FlowWorkspaceRunnable { runnable_path: path, runnable_is_flow: false }); + } + FlowModuleValue::Flow { path, .. } => { + result.push(FlowWorkspaceRunnable { runnable_path: path, runnable_is_flow: true }); + } + FlowModuleValue::ForloopFlow { modules, .. } => { + result.extend(get_flow_workspace_runnables_from_modules(modules)?); + } + FlowModuleValue::WhileloopFlow { modules, .. } => { + result.extend(get_flow_workspace_runnables_from_modules(modules)?); + } + FlowModuleValue::BranchOne { branches, .. } => { + for branch in branches { + result.extend(get_flow_workspace_runnables_from_modules(branch.modules)?); + } + } + FlowModuleValue::BranchAll { branches, .. } => { + for branch in branches { + result.extend(get_flow_workspace_runnables_from_modules(branch.modules)?); + } + } + FlowModuleValue::Identity { .. } + | FlowModuleValue::RawScript { .. } + | FlowModuleValue::FlowScript { .. } => { + // ignore + } + } + } + + Ok(result) +} + +async fn create_flow_workspace_runnables<'c>( + tx: &mut Transaction<'c, Postgres>, + path: &str, + w_id: &str, + modules_value: Option<&serde_json::Value>, + delete_existing: bool, +) -> Result<()> { + if delete_existing { + sqlx::query!( + "DELETE FROM flow_workspace_runnables WHERE flow_path = $1 AND workspace_id = $2", + path, + w_id + ) + .execute(&mut **tx) + .await?; + } + + let modules = serde_json::from_value::>( + modules_value + .ok_or_else(|| { + Error::BadRequest("Modules not found in the flow module value".to_string()) + })? + .clone(), + )?; + + let workspace_runnables = get_flow_workspace_runnables_from_modules(modules)?; + + for runnable in workspace_runnables { + sqlx::query!( + "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, $4)", + path, + runnable.runnable_path, + runnable.runnable_is_flow, + w_id + ) + .execute(&mut **tx) + .await?; + } + + Ok(()) +} + +async fn list_paths_from_workspace_runnable( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, StripPath)>, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let runnables = sqlx::query_scalar!( + r#"SELECT f.path + FROM flow_workspace_runnables fwr + JOIN flow f + ON fwr.flow_path = f.path AND fwr.workspace_id = f.workspace_id + WHERE fwr.runnable_path = $1 AND fwr.runnable_is_flow = $2 AND fwr.workspace_id = $3"#, + path.to_path(), + matches!(runnable_kind, RunnableKind::Flow), + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(runnables)) +} + async fn create_flow( authed: ApiAuthed, Extension(db): Extension, @@ -379,6 +493,9 @@ async fn create_flow( .execute(&mut *tx) .await?; + create_flow_workspace_runnables(&mut tx, &nf.path, &w_id, nf.value.get("modules"), false) + .await?; + let version = sqlx::query_scalar!( "INSERT INTO flow_version (workspace_id, path, value, schema, created_by) VALUES ($1, $2, $3, $4::text::json, $5) @@ -771,6 +888,15 @@ async fn update_flow( .await?; } + create_flow_workspace_runnables( + &mut tx, + &nf.path, + &w_id, + nf.value.get("modules"), + !is_new_path, + ) + .await?; + let version = sqlx::query_scalar!( "INSERT INTO flow_version (workspace_id, path, value, schema, created_by) VALUES ($1, $2, $3, $4::text::json, $5) RETURNING id", w_id, diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 3760fafb613de..b2b5848c32f3c 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -2478,6 +2478,22 @@ async fn update_username_in_workpsace<'c>( .execute(&mut **tx) .await?; + sqlx::query!( + r#"UPDATE flow_workspace_runnables SET flow_path = REGEXP_REPLACE(flow_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE flow_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE flow_workspace_runnables SET runnable_path = REGEXP_REPLACE(runnable_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE runnable_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + sqlx::query!( r#"UPDATE flow_node SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, new_username, diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index 439944e2b229f..1da91ae44833b 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -29,6 +29,13 @@ pub struct WithStarredInfoQuery { pub with_starred_info: Option, } +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum RunnableKind { + Script, + Flow, +} + pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> { let is_admin = is_super_admin_email(db, email).await?; diff --git a/backend/windmill-api/src/workspaces_extra.rs b/backend/windmill-api/src/workspaces_extra.rs index 2378e85c0271d..6e49d6b7da898 100644 --- a/backend/windmill-api/src/workspaces_extra.rs +++ b/backend/windmill-api/src/workspaces_extra.rs @@ -202,6 +202,14 @@ pub(crate) async fn change_workspace_id( .execute(&mut *tx) .await?; + sqlx::query!( + "UPDATE flow_workspace_runnables SET workspace_id = $1 WHERE workspace_id = $2", + &rw.new_id, + &old_id + ) + .execute(&mut *tx) + .await?; + sqlx::query!( "UPDATE flow_node SET workspace_id = $1 WHERE workspace_id = $2", &rw.new_id, From be03746f0e4cc812848c2c999d5590186c87347e Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Mon, 24 Feb 2025 20:35:10 +0100 Subject: [PATCH 2/8] track script hash --- ...ef5878f606591007c958811de9b019f7a5bc6.json} | 5 +++-- ...50205131521_flow_workspace_runnables.up.sql | 1 + backend/windmill-api/src/flows.rs | 18 ++++++++++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) rename backend/.sqlx/{query-62986d32e7c0426bbfc2238e1fc7ba20fe2ae7990db7bec0d44b9f2a9df3792b.json => query-75fe3f1868c096295ce17742497ef5878f606591007c958811de9b019f7a5bc6.json} (60%) diff --git a/backend/.sqlx/query-62986d32e7c0426bbfc2238e1fc7ba20fe2ae7990db7bec0d44b9f2a9df3792b.json b/backend/.sqlx/query-75fe3f1868c096295ce17742497ef5878f606591007c958811de9b019f7a5bc6.json similarity index 60% rename from backend/.sqlx/query-62986d32e7c0426bbfc2238e1fc7ba20fe2ae7990db7bec0d44b9f2a9df3792b.json rename to backend/.sqlx/query-75fe3f1868c096295ce17742497ef5878f606591007c958811de9b019f7a5bc6.json index 5d0d7126756aa..5cbdc7abe706c 100644 --- a/backend/.sqlx/query-62986d32e7c0426bbfc2238e1fc7ba20fe2ae7990db7bec0d44b9f2a9df3792b.json +++ b/backend/.sqlx/query-75fe3f1868c096295ce17742497ef5878f606591007c958811de9b019f7a5bc6.json @@ -1,17 +1,18 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, $4)", + "query": "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, $4, $5)", "describe": { "columns": [], "parameters": { "Left": [ "Varchar", "Varchar", + "Int8", "Bool", "Varchar" ] }, "nullable": [] }, - "hash": "62986d32e7c0426bbfc2238e1fc7ba20fe2ae7990db7bec0d44b9f2a9df3792b" + "hash": "75fe3f1868c096295ce17742497ef5878f606591007c958811de9b019f7a5bc6" } diff --git a/backend/migrations/20250205131521_flow_workspace_runnables.up.sql b/backend/migrations/20250205131521_flow_workspace_runnables.up.sql index c7f6c35e505d8..34ed5431fcd70 100644 --- a/backend/migrations/20250205131521_flow_workspace_runnables.up.sql +++ b/backend/migrations/20250205131521_flow_workspace_runnables.up.sql @@ -1,6 +1,7 @@ CREATE TABLE flow_workspace_runnables ( flow_path VARCHAR(255) NOT NULL, runnable_path VARCHAR(255) NOT NULL, + script_hash BIGINT NULL, runnable_is_flow BOOLEAN NOT NULL, workspace_id VARCHAR(50) NOT NULL, FOREIGN KEY (workspace_id, flow_path) REFERENCES flow (workspace_id, path) ON DELETE CASCADE diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index fb5c07fb86c7a..9d598dae48ea6 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -333,6 +333,7 @@ async fn check_path_conflict<'c>( struct FlowWorkspaceRunnable { runnable_path: String, runnable_is_flow: bool, + script_hash: Option, } fn get_flow_workspace_runnables_from_modules( @@ -341,11 +342,19 @@ fn get_flow_workspace_runnables_from_modules( let mut result = Vec::new(); for m in modules { match m.get_value()? { - FlowModuleValue::Script { path, .. } => { - result.push(FlowWorkspaceRunnable { runnable_path: path, runnable_is_flow: false }); + FlowModuleValue::Script { path, hash, .. } => { + result.push(FlowWorkspaceRunnable { + runnable_path: path, + runnable_is_flow: false, + script_hash: hash.map(|h| h.0), + }); } FlowModuleValue::Flow { path, .. } => { - result.push(FlowWorkspaceRunnable { runnable_path: path, runnable_is_flow: true }); + result.push(FlowWorkspaceRunnable { + runnable_path: path, + runnable_is_flow: true, + script_hash: None, + }); } FlowModuleValue::ForloopFlow { modules, .. } => { result.extend(get_flow_workspace_runnables_from_modules(modules)?); @@ -403,9 +412,10 @@ async fn create_flow_workspace_runnables<'c>( for runnable in workspace_runnables { sqlx::query!( - "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, $4)", + "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, $4, $5)", path, runnable.runnable_path, + runnable.script_hash, runnable.runnable_is_flow, w_id ) From d4a38290a099b4ab1d0b38bbce95f27710b3e8c3 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Mon, 24 Feb 2025 20:43:06 +0100 Subject: [PATCH 3/8] weird --- ....down.sql => 20250224131521_flow_workspace_runnables.down.sql} | 0 ...bles.up.sql => 20250224131521_flow_workspace_runnables.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename backend/migrations/{20250205131521_flow_workspace_runnables.down.sql => 20250224131521_flow_workspace_runnables.down.sql} (100%) rename backend/migrations/{20250205131521_flow_workspace_runnables.up.sql => 20250224131521_flow_workspace_runnables.up.sql} (100%) diff --git a/backend/migrations/20250205131521_flow_workspace_runnables.down.sql b/backend/migrations/20250224131521_flow_workspace_runnables.down.sql similarity index 100% rename from backend/migrations/20250205131521_flow_workspace_runnables.down.sql rename to backend/migrations/20250224131521_flow_workspace_runnables.down.sql diff --git a/backend/migrations/20250205131521_flow_workspace_runnables.up.sql b/backend/migrations/20250224131521_flow_workspace_runnables.up.sql similarity index 100% rename from backend/migrations/20250205131521_flow_workspace_runnables.up.sql rename to backend/migrations/20250224131521_flow_workspace_runnables.up.sql From 3a861cd9ac808d92e13e98c556f655e8cb595a2b Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Tue, 25 Feb 2025 10:22:44 +0100 Subject: [PATCH 4/8] do it with lock --- ...a6ed16c25aea4584103c1b9a3651eb31bfe53.json | 16 +++ ...06f62e700695810f23108e975dfcd1fee7a3.json} | 5 +- ...0224131521_flow_workspace_runnables.up.sql | 2 + backend/windmill-api/src/flows.rs | 118 ++---------------- .../windmill-worker/src/worker_lockfiles.rs | 21 ++++ 5 files changed, 49 insertions(+), 113 deletions(-) create mode 100644 backend/.sqlx/query-35795d27c4ca69d2f145b4dba08a6ed16c25aea4584103c1b9a3651eb31bfe53.json rename backend/.sqlx/{query-75fe3f1868c096295ce17742497ef5878f606591007c958811de9b019f7a5bc6.json => query-c35f44f91b08fa57e29a2b4a685706f62e700695810f23108e975dfcd1fee7a3.json} (73%) diff --git a/backend/.sqlx/query-35795d27c4ca69d2f145b4dba08a6ed16c25aea4584103c1b9a3651eb31bfe53.json b/backend/.sqlx/query-35795d27c4ca69d2f145b4dba08a6ed16c25aea4584103c1b9a3651eb31bfe53.json new file mode 100644 index 0000000000000..06730d3e26760 --- /dev/null +++ b/backend/.sqlx/query-35795d27c4ca69d2f145b4dba08a6ed16c25aea4584103c1b9a3651eb31bfe53.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, runnable_is_flow, workspace_id) VALUES ($1, $2, TRUE, $3) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "35795d27c4ca69d2f145b4dba08a6ed16c25aea4584103c1b9a3651eb31bfe53" +} diff --git a/backend/.sqlx/query-75fe3f1868c096295ce17742497ef5878f606591007c958811de9b019f7a5bc6.json b/backend/.sqlx/query-c35f44f91b08fa57e29a2b4a685706f62e700695810f23108e975dfcd1fee7a3.json similarity index 73% rename from backend/.sqlx/query-75fe3f1868c096295ce17742497ef5878f606591007c958811de9b019f7a5bc6.json rename to backend/.sqlx/query-c35f44f91b08fa57e29a2b4a685706f62e700695810f23108e975dfcd1fee7a3.json index 5cbdc7abe706c..77e89c52a850e 100644 --- a/backend/.sqlx/query-75fe3f1868c096295ce17742497ef5878f606591007c958811de9b019f7a5bc6.json +++ b/backend/.sqlx/query-c35f44f91b08fa57e29a2b4a685706f62e700695810f23108e975dfcd1fee7a3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, $4, $5)", + "query": "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, FALSE, $4) ON CONFLICT DO NOTHING", "describe": { "columns": [], "parameters": { @@ -8,11 +8,10 @@ "Varchar", "Varchar", "Int8", - "Bool", "Varchar" ] }, "nullable": [] }, - "hash": "75fe3f1868c096295ce17742497ef5878f606591007c958811de9b019f7a5bc6" + "hash": "c35f44f91b08fa57e29a2b4a685706f62e700695810f23108e975dfcd1fee7a3" } diff --git a/backend/migrations/20250224131521_flow_workspace_runnables.up.sql b/backend/migrations/20250224131521_flow_workspace_runnables.up.sql index 34ed5431fcd70..fe61497c6df36 100644 --- a/backend/migrations/20250224131521_flow_workspace_runnables.up.sql +++ b/backend/migrations/20250224131521_flow_workspace_runnables.up.sql @@ -7,4 +7,6 @@ CREATE TABLE flow_workspace_runnables ( FOREIGN KEY (workspace_id, flow_path) REFERENCES flow (workspace_id, path) ON DELETE CASCADE ); +CREATE UNIQUE INDEX flow_workspace_without_hash_unique_idx ON flow_workspace_runnables (flow_path, runnable_path, runnable_is_flow, workspace_id) WHERE script_hash IS NULL; +CREATE UNIQUE INDEX flow_workspace_with_hash_unique_idx ON flow_workspace_runnables (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) WHERE script_hash IS NOT NULL; CREATE INDEX flow_workspace_runnable_path_is_flow_idx ON flow_workspace_runnables (runnable_path, runnable_is_flow, workspace_id); diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 9d598dae48ea6..072dcb4c93413 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -33,7 +33,6 @@ use sql_builder::prelude::*; use sqlx::{FromRow, Postgres, Transaction}; use windmill_audit::audit_ee::audit_log; use windmill_audit::ActionKind; -use windmill_common::flows::{FlowModule, FlowModuleValue}; use windmill_common::utils::query_elems_from_hub; use windmill_common::worker::to_raw_value; use windmill_common::HUB_BASE_URL; @@ -329,103 +328,6 @@ async fn check_path_conflict<'c>( return Ok(()); } -#[derive(Debug, Deserialize, Serialize)] -struct FlowWorkspaceRunnable { - runnable_path: String, - runnable_is_flow: bool, - script_hash: Option, -} - -fn get_flow_workspace_runnables_from_modules( - modules: Vec, -) -> Result> { - let mut result = Vec::new(); - for m in modules { - match m.get_value()? { - FlowModuleValue::Script { path, hash, .. } => { - result.push(FlowWorkspaceRunnable { - runnable_path: path, - runnable_is_flow: false, - script_hash: hash.map(|h| h.0), - }); - } - FlowModuleValue::Flow { path, .. } => { - result.push(FlowWorkspaceRunnable { - runnable_path: path, - runnable_is_flow: true, - script_hash: None, - }); - } - FlowModuleValue::ForloopFlow { modules, .. } => { - result.extend(get_flow_workspace_runnables_from_modules(modules)?); - } - FlowModuleValue::WhileloopFlow { modules, .. } => { - result.extend(get_flow_workspace_runnables_from_modules(modules)?); - } - FlowModuleValue::BranchOne { branches, .. } => { - for branch in branches { - result.extend(get_flow_workspace_runnables_from_modules(branch.modules)?); - } - } - FlowModuleValue::BranchAll { branches, .. } => { - for branch in branches { - result.extend(get_flow_workspace_runnables_from_modules(branch.modules)?); - } - } - FlowModuleValue::Identity { .. } - | FlowModuleValue::RawScript { .. } - | FlowModuleValue::FlowScript { .. } => { - // ignore - } - } - } - - Ok(result) -} - -async fn create_flow_workspace_runnables<'c>( - tx: &mut Transaction<'c, Postgres>, - path: &str, - w_id: &str, - modules_value: Option<&serde_json::Value>, - delete_existing: bool, -) -> Result<()> { - if delete_existing { - sqlx::query!( - "DELETE FROM flow_workspace_runnables WHERE flow_path = $1 AND workspace_id = $2", - path, - w_id - ) - .execute(&mut **tx) - .await?; - } - - let modules = serde_json::from_value::>( - modules_value - .ok_or_else(|| { - Error::BadRequest("Modules not found in the flow module value".to_string()) - })? - .clone(), - )?; - - let workspace_runnables = get_flow_workspace_runnables_from_modules(modules)?; - - for runnable in workspace_runnables { - sqlx::query!( - "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, $4, $5)", - path, - runnable.runnable_path, - runnable.script_hash, - runnable.runnable_is_flow, - w_id - ) - .execute(&mut **tx) - .await?; - } - - Ok(()) -} - async fn list_paths_from_workspace_runnable( authed: ApiAuthed, Extension(user_db): Extension, @@ -503,9 +405,6 @@ async fn create_flow( .execute(&mut *tx) .await?; - create_flow_workspace_runnables(&mut tx, &nf.path, &w_id, nf.value.get("modules"), false) - .await?; - let version = sqlx::query_scalar!( "INSERT INTO flow_version (workspace_id, path, value, schema, created_by) VALUES ($1, $2, $3, $4::text::json, $5) @@ -896,17 +795,16 @@ async fn update_flow( ) .execute(&mut *tx) .await?; + } else { + sqlx::query!( + "DELETE FROM flow_workspace_runnables WHERE flow_path = $1 AND workspace_id = $2", + flow_path, + w_id + ) + .execute(&mut *tx) + .await?; } - create_flow_workspace_runnables( - &mut tx, - &nf.path, - &w_id, - nf.value.get("modules"), - !is_new_path, - ) - .await?; - let version = sqlx::query_scalar!( "INSERT INTO flow_version (workspace_id, path, value, schema, created_by) VALUES ($1, $2, $3, $4::text::json, $5) RETURNING id", w_id, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index f215a6783c7ff..795f3e381b25a 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -981,6 +981,27 @@ async fn lock_modules<'c>( } .into(); } + FlowModuleValue::Script { path, hash, .. } => { + sqlx::query!( + "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, FALSE, $4) ON CONFLICT DO NOTHING", + job_path, + path, + hash.map(|h| h.0), + job.workspace_id + ) + .execute(&mut *tx) + .await?; + } + FlowModuleValue::Flow { path, .. } => { + sqlx::query!( + "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, runnable_is_flow, workspace_id) VALUES ($1, $2, TRUE, $3) ON CONFLICT DO NOTHING", + job_path, + path, + job.workspace_id + ) + .execute(&mut *tx) + .await?; + } _ => (), }; modified_ids.extend(nmodified_ids); From bb8f7098940626ae420b8c5ef707f22a2256cd30 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Tue, 25 Feb 2025 11:29:34 +0100 Subject: [PATCH 5/8] Revert "feat: add support for | None and Optional in python (#5361)" This reverts commit 9736355d5f82615100212698c5537997e5a0de39. --- backend/parsers/windmill-parser-py/src/lib.rs | 84 +++---------------- frontend/package-lock.json | 8 +- frontend/package.json | 2 +- 3 files changed, 17 insertions(+), 77 deletions(-) diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index f5f9574888d55..76f0c415ea1e7 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -96,11 +96,11 @@ pub fn parse_python_signature( .iter() .enumerate() .map(|(i, x)| { - let (mut typ, has_default) = x + let mut typ = x .as_arg() .annotation .as_ref() - .map_or((Typ::Unknown, false), |e| parse_expr(e)); + .map_or(Typ::Unknown, |e| parse_expr(e)); let default = if i >= def_arg_start { params @@ -138,7 +138,7 @@ pub fn parse_python_signature( otyp: None, name: x.as_arg().arg.to_string(), typ, - has_default: has_default || default.is_some(), + has_default: default.is_some(), default, oidx: None, } @@ -158,27 +158,17 @@ pub fn parse_python_signature( } } -fn parse_expr(e: &Box) -> (Typ, bool) { +fn parse_expr(e: &Box) -> Typ { match e.as_ref() { - Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref()), false), + Expr::Name(ExprName { id, .. }) => parse_typ(id.as_ref()), Expr::Attribute(x) => { if x.value .as_name_expr() .is_some_and(|x| x.id.as_str() == "wmill") { - (parse_typ(x.attr.as_str()), false) + parse_typ(x.attr.as_str()) } else { - (Typ::Unknown, false) - } - } - Expr::BinOp(x) => { - if matches!( - x.right.as_ref(), - Expr::Constant(ExprConstant { value: Constant::None, .. }) - ) { - (parse_expr(&x.left).0, true) - } else { - (Typ::Unknown, false) + Typ::Unknown } } Expr::Subscript(x) => match x.value.as_ref() { @@ -203,15 +193,14 @@ fn parse_expr(e: &Box) -> (Typ, bool) { } _ => None, }; - (Typ::Str(values), false) + Typ::Str(values) } - "List" => (Typ::List(Box::new(parse_expr(&x.slice).0)), false), - "Optional" => (parse_expr(&x.slice).0, true), - _ => (Typ::Unknown, false), + "List" => Typ::List(Box::new(parse_expr(&x.slice))), + _ => Typ::Unknown, }, - _ => (Typ::Unknown, false), + _ => Typ::Unknown, }, - _ => (Typ::Unknown, false), + _ => Typ::Unknown, } } @@ -687,53 +676,4 @@ def main(a: list, e: List[int], b: list = [1,2,3,4], c = [1,2,3,4], d = ["a", "b Ok(()) } - - #[test] - fn test_parse_python_sig_9() -> anyhow::Result<()> { - let code = r#" -from typing import Optional -def main(a: str, b: Optional[str], c: str | None): return -"#; - println!( - "{}", - serde_json::to_string(&parse_python_signature(code, None, false)?)? - ); - assert_eq!( - parse_python_signature(code, None, false)?, - MainArgSignature { - star_args: false, - star_kwargs: false, - args: vec![ - Arg { - otyp: None, - name: "a".to_string(), - typ: Typ::Str(None), - default: None, - has_default: false, - oidx: None - }, - Arg { - otyp: None, - name: "b".to_string(), - typ: Typ::Str(None), - default: None, - has_default: true, - oidx: None - }, - Arg { - otyp: None, - name: "c".to_string(), - typ: Typ::Str(None), - default: None, - has_default: true, - oidx: None - }, - ], - no_main_func: Some(false), - has_preprocessor: Some(false) - } - ); - - Ok(()) - } } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 35bcec1bc9a6d..803aceffb437d 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -70,7 +70,7 @@ "windmill-parser-wasm-csharp": "^1.437.1", "windmill-parser-wasm-go": "^1.429.0", "windmill-parser-wasm-php": "^1.429.0", - "windmill-parser-wasm-py": "^1.467.1", + "windmill-parser-wasm-py": "^1.429.0", "windmill-parser-wasm-regex": "^1.439.0", "windmill-parser-wasm-rust": "^1.429.0", "windmill-parser-wasm-ts": "^1.429.0", @@ -12594,9 +12594,9 @@ "integrity": "sha512-SGJAtNpfdRZftkGboxWsm/yQDnJBJodwPQUbX2cWk/aoNook6ULesZwsYtBC9WN1VH6TIskLiVPohMmu6jtXmw==" }, "node_modules/windmill-parser-wasm-py": { - "version": "1.467.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.467.1.tgz", - "integrity": "sha512-Px/UvNjCCScf5V7B90PirxYe7Mn/zMnhk++cLuvlg08j45GR/mzFOIfC7Xk9e555ppX3gTlCk+auH4aIlG2tVw==" + "version": "1.429.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.429.0.tgz", + "integrity": "sha512-cqc+tblQVHVrc8wNA4esVYD1Dv59XQJ4mHXFFPa8Lx5UjXv7FO/0VQxyQuRyXaP/V6J6DDy0oeN107eFNLxrBg==" }, "node_modules/windmill-parser-wasm-regex": { "version": "1.439.0", diff --git a/frontend/package.json b/frontend/package.json index cd2d5ecff288b..88bae0312c851 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -145,7 +145,7 @@ "windmill-parser-wasm-csharp": "^1.437.1", "windmill-parser-wasm-go": "^1.429.0", "windmill-parser-wasm-php": "^1.429.0", - "windmill-parser-wasm-py": "^1.467.1", + "windmill-parser-wasm-py": "^1.429.0", "windmill-parser-wasm-regex": "^1.439.0", "windmill-parser-wasm-rust": "^1.429.0", "windmill-parser-wasm-ts": "^1.429.0", From 9a3b0d2f16496ed1f9de6823cc9f620a796a59a4 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Tue, 25 Feb 2025 11:34:15 +0100 Subject: [PATCH 6/8] Revert "Revert "feat: add support for | None and Optional in python (#5361)"" This reverts commit bb8f7098940626ae420b8c5ef707f22a2256cd30. --- backend/parsers/windmill-parser-py/src/lib.rs | 84 ++++++++++++++++--- frontend/package-lock.json | 8 +- frontend/package.json | 2 +- 3 files changed, 77 insertions(+), 17 deletions(-) diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 76f0c415ea1e7..f5f9574888d55 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -96,11 +96,11 @@ pub fn parse_python_signature( .iter() .enumerate() .map(|(i, x)| { - let mut typ = x + let (mut typ, has_default) = x .as_arg() .annotation .as_ref() - .map_or(Typ::Unknown, |e| parse_expr(e)); + .map_or((Typ::Unknown, false), |e| parse_expr(e)); let default = if i >= def_arg_start { params @@ -138,7 +138,7 @@ pub fn parse_python_signature( otyp: None, name: x.as_arg().arg.to_string(), typ, - has_default: default.is_some(), + has_default: has_default || default.is_some(), default, oidx: None, } @@ -158,17 +158,27 @@ pub fn parse_python_signature( } } -fn parse_expr(e: &Box) -> Typ { +fn parse_expr(e: &Box) -> (Typ, bool) { match e.as_ref() { - Expr::Name(ExprName { id, .. }) => parse_typ(id.as_ref()), + Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref()), false), Expr::Attribute(x) => { if x.value .as_name_expr() .is_some_and(|x| x.id.as_str() == "wmill") { - parse_typ(x.attr.as_str()) + (parse_typ(x.attr.as_str()), false) } else { - Typ::Unknown + (Typ::Unknown, false) + } + } + Expr::BinOp(x) => { + if matches!( + x.right.as_ref(), + Expr::Constant(ExprConstant { value: Constant::None, .. }) + ) { + (parse_expr(&x.left).0, true) + } else { + (Typ::Unknown, false) } } Expr::Subscript(x) => match x.value.as_ref() { @@ -193,14 +203,15 @@ fn parse_expr(e: &Box) -> Typ { } _ => None, }; - Typ::Str(values) + (Typ::Str(values), false) } - "List" => Typ::List(Box::new(parse_expr(&x.slice))), - _ => Typ::Unknown, + "List" => (Typ::List(Box::new(parse_expr(&x.slice).0)), false), + "Optional" => (parse_expr(&x.slice).0, true), + _ => (Typ::Unknown, false), }, - _ => Typ::Unknown, + _ => (Typ::Unknown, false), }, - _ => Typ::Unknown, + _ => (Typ::Unknown, false), } } @@ -676,4 +687,53 @@ def main(a: list, e: List[int], b: list = [1,2,3,4], c = [1,2,3,4], d = ["a", "b Ok(()) } + + #[test] + fn test_parse_python_sig_9() -> anyhow::Result<()> { + let code = r#" +from typing import Optional +def main(a: str, b: Optional[str], c: str | None): return +"#; + println!( + "{}", + serde_json::to_string(&parse_python_signature(code, None, false)?)? + ); + assert_eq!( + parse_python_signature(code, None, false)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![ + Arg { + otyp: None, + name: "a".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None + }, + Arg { + otyp: None, + name: "b".to_string(), + typ: Typ::Str(None), + default: None, + has_default: true, + oidx: None + }, + Arg { + otyp: None, + name: "c".to_string(), + typ: Typ::Str(None), + default: None, + has_default: true, + oidx: None + }, + ], + no_main_func: Some(false), + has_preprocessor: Some(false) + } + ); + + Ok(()) + } } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 803aceffb437d..35bcec1bc9a6d 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -70,7 +70,7 @@ "windmill-parser-wasm-csharp": "^1.437.1", "windmill-parser-wasm-go": "^1.429.0", "windmill-parser-wasm-php": "^1.429.0", - "windmill-parser-wasm-py": "^1.429.0", + "windmill-parser-wasm-py": "^1.467.1", "windmill-parser-wasm-regex": "^1.439.0", "windmill-parser-wasm-rust": "^1.429.0", "windmill-parser-wasm-ts": "^1.429.0", @@ -12594,9 +12594,9 @@ "integrity": "sha512-SGJAtNpfdRZftkGboxWsm/yQDnJBJodwPQUbX2cWk/aoNook6ULesZwsYtBC9WN1VH6TIskLiVPohMmu6jtXmw==" }, "node_modules/windmill-parser-wasm-py": { - "version": "1.429.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.429.0.tgz", - "integrity": "sha512-cqc+tblQVHVrc8wNA4esVYD1Dv59XQJ4mHXFFPa8Lx5UjXv7FO/0VQxyQuRyXaP/V6J6DDy0oeN107eFNLxrBg==" + "version": "1.467.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.467.1.tgz", + "integrity": "sha512-Px/UvNjCCScf5V7B90PirxYe7Mn/zMnhk++cLuvlg08j45GR/mzFOIfC7Xk9e555ppX3gTlCk+auH4aIlG2tVw==" }, "node_modules/windmill-parser-wasm-regex": { "version": "1.439.0", diff --git a/frontend/package.json b/frontend/package.json index 88bae0312c851..cd2d5ecff288b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -145,7 +145,7 @@ "windmill-parser-wasm-csharp": "^1.437.1", "windmill-parser-wasm-go": "^1.429.0", "windmill-parser-wasm-php": "^1.429.0", - "windmill-parser-wasm-py": "^1.429.0", + "windmill-parser-wasm-py": "^1.467.1", "windmill-parser-wasm-regex": "^1.439.0", "windmill-parser-wasm-rust": "^1.429.0", "windmill-parser-wasm-ts": "^1.429.0", From e87d27b7a2a580de4eadd7f5874aae29e5bfd0f7 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Tue, 25 Feb 2025 12:16:23 +0100 Subject: [PATCH 7/8] update openapi --- backend/windmill-api/openapi.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 5aaec60cde9fb..3d0825632e073 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -5292,6 +5292,26 @@ paths: schema: $ref: "#/components/schemas/FlowVersion" + /w/{workspace}/flows/list_paths_from_workspace_runnable/{runnable_kind}/{path}: + get: + summary: list flow paths from workspace runnable + operationId: listFlowPathsFromWorkspaceRunnable + tags: + - flow + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/RunnableKind" + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: list of flow paths + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/flows/get/v/{version}/p/{path}: get: summary: get flow version From a1554900f475e1a0c2bf976eb5a73c1ea638dae3 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Tue, 25 Feb 2025 22:16:24 +0100 Subject: [PATCH 8/8] delete old in lock_modules + don't track hub scripts --- backend/windmill-api/src/flows.rs | 8 -------- backend/windmill-worker/src/worker_lockfiles.rs | 9 ++++++++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 072dcb4c93413..8cd65dd55a432 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -795,14 +795,6 @@ async fn update_flow( ) .execute(&mut *tx) .await?; - } else { - sqlx::query!( - "DELETE FROM flow_workspace_runnables WHERE flow_path = $1 AND workspace_id = $2", - flow_path, - w_id - ) - .execute(&mut *tx) - .await?; } let version = sqlx::query_scalar!( diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 795f3e381b25a..5df02daa57f92 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -620,6 +620,13 @@ pub async fn handle_flow_dependency_job( tx = clear_dependency_parent_path(&parent_path, &job_path, &job.workspace_id, "flow", tx) .await?; + sqlx::query!( + "DELETE FROM flow_workspace_runnables WHERE flow_path = $1 AND workspace_id = $2", + job_path, + job.workspace_id + ) + .execute(&mut *tx) + .await?; let modified_ids; let errors; (flow.modules, tx, modified_ids, errors) = lock_modules( @@ -981,7 +988,7 @@ async fn lock_modules<'c>( } .into(); } - FlowModuleValue::Script { path, hash, .. } => { + FlowModuleValue::Script { path, hash, .. } if !path.starts_with("hub/") => { sqlx::query!( "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, FALSE, $4) ON CONFLICT DO NOTHING", job_path,