From 8289c52d0544908addffc983c2f8ca6ae40236ca Mon Sep 17 00:00:00 2001 From: Christian Meesters Date: Thu, 2 May 2024 11:21:52 +0200 Subject: [PATCH 1/3] feat: new function to warn when running in a job --- snakemake_executor_plugin_slurm/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/snakemake_executor_plugin_slurm/__init__.py b/snakemake_executor_plugin_slurm/__init__.py index 0fae7dd2..d4515334 100644 --- a/snakemake_executor_plugin_slurm/__init__.py +++ b/snakemake_executor_plugin_slurm/__init__.py @@ -49,11 +49,22 @@ # Implementation of your executor class Executor(RemoteExecutor): def __post_init__(self): + # run check whether we are running in a SLURM job context + self.warn_on_jobcontext() self.run_uuid = str(uuid.uuid4()) self.logger.info(f"SLURM run ID: {self.run_uuid}") self._fallback_account_arg = None self._fallback_partition = None + def warn_on_jobcontext(self, done=None): + if not done: + if "SLURM_JOB_ID" in os.environ: + self.logger.warning( + "Running Snakemake in a SLURM within a job may lead to unexpected behavior. Please run Snakemake directly on the head node." + ) + time.sleep(5) + done = True + def additional_general_args(self): return "--executor slurm-jobstep --jobs 1" From 0a02507ebe1817b46ffb994f57c8af7bfe95840e Mon Sep 17 00:00:00 2001 From: Christian Meesters Date: Thu, 2 May 2024 11:43:57 +0200 Subject: [PATCH 2/3] docs: documented the intended behaviour and the just implemented warning --- docs/further.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/further.md b/docs/further.md index 48aa3522..b11eeccf 100644 --- a/docs/further.md +++ b/docs/further.md @@ -2,7 +2,7 @@ ## The general Idea -To use this plugin, log in to your cluster's head node (sometimes called the "login" node), activate your environment as usual and start Snakemake. Snakemake will then submit your jobs as cluster jobs. +To use this plugin, log in to your cluster's head node (sometimes called the "login" node), activate your environment as usual, and start Snakemake. Snakemake will then submit your jobs as cluster jobs. ## Specifying Account and Partition @@ -220,6 +220,12 @@ export SNAKEMAKE_PROFILE="$HOME/.config/snakemake" Further note, that there is further development ongoing to enable differentiation of file access patterns. +## Nesting Jobs (or Running this Plugin within a Job) + +Some environments provide a shell within a SLURM job, for instance, IDEs started in on-demand context. If Snakemake attempts to use this plugin to spawn jobs on the cluster, this may work just as intended. Or it might not: depending on cluster settings or individual settings, submitted jobs may be ill-parameterized or will not find the right environment. + +If the plugin detects to be running within a job, it will therefore issue a warning and stop for 5 seconds. + ## Summary: When put together, a frequent command line looks like: From dba9196a68c4bfab91a25cea687084537b8617db Mon Sep 17 00:00:00 2001 From: Christian Meesters Date: Thu, 2 May 2024 12:07:10 +0200 Subject: [PATCH 3/3] fix: format --- snakemake_executor_plugin_slurm/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/snakemake_executor_plugin_slurm/__init__.py b/snakemake_executor_plugin_slurm/__init__.py index c1fe4a1b..6913035d 100644 --- a/snakemake_executor_plugin_slurm/__init__.py +++ b/snakemake_executor_plugin_slurm/__init__.py @@ -60,7 +60,9 @@ def warn_on_jobcontext(self, done=None): if not done: if "SLURM_JOB_ID" in os.environ: self.logger.warning( - "Running Snakemake in a SLURM within a job may lead to unexpected behavior. Please run Snakemake directly on the head node." + "Running Snakemake in a SLURM within a job may lead" + " to unexpected behavior. Please run Snakemake directly" + " on the head node." ) time.sleep(5) done = True