feat: remove pip fallback option for python and ansible (#5186)

* refactor!: Remove `pip` fallback option for python and ansible BREAKING CHANGE: pip was deprecated since 1.425.0 (2024-11-15) * fix errors in main.rs * fix tests * remove nsjail for pip * fix imports * fix compilation error
windmill-labs · Feb 19, 2025 · 50bc4ca · 50bc4ca
1 parent 35d5293
commit 50bc4ca
Show file tree

Hide file tree

Showing 10 changed files with 78 additions and 449 deletions.
diff --git a/backend/src/main.rs b/backend/src/main.rs
@@ -65,9 +65,9 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
 use windmill_worker::{
     get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR,
     DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR,
-    LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR,
-    PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR,
-    TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, UV_CACHE_DIR,
+    POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR,
+    RUST_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR,
+    TAR_PY313_CACHE_DIR, UV_CACHE_DIR,
 };
 
 use crate::monitor::{
@@ -1041,10 +1041,8 @@ pub async fn run_workers(
     let mut handles = Vec::with_capacity(num_workers as usize);
 
     for x in [
-        LOCK_CACHE_DIR,
         TMP_LOGS_DIR,
         UV_CACHE_DIR,
-        TAR_PIP_CACHE_DIR,
         DENO_CACHE_DIR,
         DENO_CACHE_DIR_DEPS,
         DENO_CACHE_DIR_NPM,
@@ -1057,7 +1055,6 @@ pub async fn run_workers(
         TAR_PY311_CACHE_DIR,
         TAR_PY312_CACHE_DIR,
         TAR_PY313_CACHE_DIR,
-        PIP_CACHE_DIR,
         BUN_BUNDLE_CACHE_DIR,
         GO_CACHE_DIR,
         GO_BIN_CACHE_DIR,

diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs
@@ -1015,15 +1015,10 @@ fn spawn_test_worker(
     tokio::sync::broadcast::Sender<()>,
     tokio::task::JoinHandle<()>,
 ) {
-    for x in [
-        windmill_worker::LOCK_CACHE_DIR,
-        windmill_worker::GO_BIN_CACHE_DIR,
-    ] {
-        std::fs::DirBuilder::new()
-            .recursive(true)
-            .create(x)
-            .expect("could not create initial worker dir");
-    }
+    std::fs::DirBuilder::new()
+        .recursive(true)
+        .create(windmill_worker::GO_BIN_CACHE_DIR)
+        .expect("could not create initial worker dir");
 
     let (tx, rx) = tokio::sync::broadcast::channel(1);
     let db = db.to_owned();

diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs
@@ -362,9 +362,6 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
 #[annotations("#")]
 pub struct PythonAnnotations {
     pub no_cache: bool,
-    pub no_uv: bool,
-    pub no_uv_install: bool,
-    pub no_uv_compile: bool,
     pub no_postinstall: bool,
     pub py310: bool,
     pub py311: bool,

diff --git a/backend/windmill-worker/nsjail/download.py.pip.config.proto b/backend/windmill-worker/nsjail/download.py.pip.config.proto
diff --git a/backend/windmill-worker/nsjail/download_deps.py.pip.sh b/backend/windmill-worker/nsjail/download_deps.py.pip.sh
diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs
@@ -86,7 +86,6 @@ async fn handle_ansible_python_deps(
                     &mut Some(occupancy_metrics),
                     PyVersion::Py311,
                     false,
-                    false,
                 )
                 .await
                 .map_err(|e| {
@@ -113,7 +112,6 @@ async fn handle_ansible_python_deps(
             worker_dir,
             &mut Some(occupancy_metrics),
             crate::python_executor::PyVersion::Py311,
-            false,
         )
         .await?;
         additional_python_paths.append(&mut venv_path);

diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs
@@ -22,23 +22,18 @@ pub async fn build_tar_and_push(
     folder: String,
     // python_311
     python_xyz: String,
-    no_uv: bool,
 ) -> error::Result<()> {
     use object_store::path::Path;
 
-    use crate::{TAR_PIP_CACHE_DIR, TAR_PYBASE_CACHE_DIR};
+    use crate::TAR_PYBASE_CACHE_DIR;
 
     tracing::info!("Started building and pushing piptar {folder}");
     let start = Instant::now();
 
     // e.g. tiny==1.0.0
     let folder_name = folder.split("/").last().unwrap();
 
-    let prefix = if no_uv {
-        TAR_PIP_CACHE_DIR
-    } else {
-        &format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz)
-    };
+    let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz);
     let tar_path = format!("{prefix}/{folder_name}_tar.tar",);
 
     let tar_file = std::fs::File::create(&tar_path)?;
@@ -59,10 +54,7 @@ pub async fn build_tar_and_push(
     // })?;
     if let Err(e) = s3_client
         .put(
-            &Path::from(format!(
-                "/tar/{TARGET}/{}/{folder_name}.tar",
-                if no_uv { "pip" } else { &python_xyz }
-            )),
+            &Path::from(format!("/tar/{TARGET}/{python_xyz}/{folder_name}.tar")),
             std::fs::read(&tar_path)?.into(),
         )
         .await
@@ -92,7 +84,6 @@ pub async fn pull_from_tar(
     folder: String,
     // python_311
     python_xyz: String,
-    no_uv: bool,
 ) -> error::Result<()> {
     use windmill_common::s3_helpers::attempt_fetch_bytes;
 
@@ -102,10 +93,7 @@ pub async fn pull_from_tar(
 
     let start = Instant::now();
 
-    let tar_path = format!(
-        "tar/{TARGET}/{}/{folder_name}.tar",
-        if no_uv { "pip".to_owned() } else { python_xyz }
-    );
+    let tar_path = format!("tar/{TARGET}/{python_xyz}/{folder_name}.tar");
     let bytes = attempt_fetch_bytes(client, &tar_path).await?;
 
     extract_tar(bytes, &folder).await.map_err(|e| {