Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: remove pip fallback option for python and ansible #5186

Merged
merged 9 commits into from
Feb 19, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions backend/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
use windmill_worker::{
get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR,
DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR,
LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR,
PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR,
TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, UV_CACHE_DIR,
POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR,
RUST_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR,
TAR_PY313_CACHE_DIR, UV_CACHE_DIR,
};

use crate::monitor::{
Expand Down Expand Up @@ -1033,10 +1033,8 @@ pub async fn run_workers(
let mut handles = Vec::with_capacity(num_workers as usize);

for x in [
LOCK_CACHE_DIR,
TMP_LOGS_DIR,
UV_CACHE_DIR,
TAR_PIP_CACHE_DIR,
DENO_CACHE_DIR,
DENO_CACHE_DIR_DEPS,
DENO_CACHE_DIR_NPM,
Expand All @@ -1049,7 +1047,6 @@ pub async fn run_workers(
TAR_PY311_CACHE_DIR,
TAR_PY312_CACHE_DIR,
TAR_PY313_CACHE_DIR,
PIP_CACHE_DIR,
BUN_BUNDLE_CACHE_DIR,
GO_CACHE_DIR,
GO_BIN_CACHE_DIR,
Expand Down
13 changes: 4 additions & 9 deletions backend/tests/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1015,15 +1015,10 @@ fn spawn_test_worker(
tokio::sync::broadcast::Sender<()>,
tokio::task::JoinHandle<()>,
) {
for x in [
windmill_worker::LOCK_CACHE_DIR,
windmill_worker::GO_BIN_CACHE_DIR,
] {
std::fs::DirBuilder::new()
.recursive(true)
.create(x)
.expect("could not create initial worker dir");
}
std::fs::DirBuilder::new()
.recursive(true)
.create(windmill_worker::GO_BIN_CACHE_DIR)
.expect("could not create initial worker dir");

let (tx, rx) = tokio::sync::broadcast::channel(1);
let db = db.to_owned();
Expand Down
3 changes: 0 additions & 3 deletions backend/windmill-common/src/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -362,9 +362,6 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
#[annotations("#")]
pub struct PythonAnnotations {
pub no_cache: bool,
pub no_uv: bool,
pub no_uv_install: bool,
pub no_uv_compile: bool,
pub no_postinstall: bool,
pub py310: bool,
pub py311: bool,
Expand Down
86 changes: 0 additions & 86 deletions backend/windmill-worker/nsjail/download.py.pip.config.proto

This file was deleted.

24 changes: 0 additions & 24 deletions backend/windmill-worker/nsjail/download_deps.py.pip.sh

This file was deleted.

2 changes: 0 additions & 2 deletions backend/windmill-worker/src/ansible_executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@ async fn handle_ansible_python_deps(
&mut Some(occupancy_metrics),
PyVersion::Py311,
false,
false,
)
.await
.map_err(|e| {
Expand All @@ -113,7 +112,6 @@ async fn handle_ansible_python_deps(
worker_dir,
&mut Some(occupancy_metrics),
crate::python_executor::PyVersion::Py311,
false,
)
.await?;
additional_python_paths.append(&mut venv_path);
Expand Down
20 changes: 4 additions & 16 deletions backend/windmill-worker/src/global_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,18 @@ pub async fn build_tar_and_push(
folder: String,
// python_311
python_xyz: String,
no_uv: bool,
) -> error::Result<()> {
use object_store::path::Path;

use crate::{TAR_PIP_CACHE_DIR, TAR_PYBASE_CACHE_DIR};
use crate::TAR_PYBASE_CACHE_DIR;

tracing::info!("Started building and pushing piptar {folder}");
let start = Instant::now();

// e.g. tiny==1.0.0
let folder_name = folder.split("/").last().unwrap();

let prefix = if no_uv {
TAR_PIP_CACHE_DIR
} else {
&format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz)
};
let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz);
let tar_path = format!("{prefix}/{folder_name}_tar.tar",);

let tar_file = std::fs::File::create(&tar_path)?;
Expand All @@ -59,10 +54,7 @@ pub async fn build_tar_and_push(
// })?;
if let Err(e) = s3_client
.put(
&Path::from(format!(
"/tar/{TARGET}/{}/{folder_name}.tar",
if no_uv { "pip" } else { &python_xyz }
)),
&Path::from(format!("/tar/{TARGET}/{python_xyz}/{folder_name}.tar")),
std::fs::read(&tar_path)?.into(),
)
.await
Expand Down Expand Up @@ -92,7 +84,6 @@ pub async fn pull_from_tar(
folder: String,
// python_311
python_xyz: String,
no_uv: bool,
) -> error::Result<()> {
use windmill_common::s3_helpers::attempt_fetch_bytes;

Expand All @@ -102,10 +93,7 @@ pub async fn pull_from_tar(

let start = Instant::now();

let tar_path = format!(
"tar/{TARGET}/{}/{folder_name}.tar",
if no_uv { "pip".to_owned() } else { python_xyz }
);
let tar_path = format!("tar/{TARGET}/{python_xyz}/{folder_name}.tar");
let bytes = attempt_fetch_bytes(client, &tar_path).await?;

extract_tar(bytes, &folder).await.map_err(|e| {
Expand Down
Loading