From b4235fb01ace6e464b84e90c1d8d8cf79e932253 Mon Sep 17 00:00:00 2001 From: Christian Meesters Date: Wed, 22 Jan 2025 08:17:33 +0100 Subject: [PATCH 1/6] feat: writing error to stderr --- .github/workflows/post_to_mastodon.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/post_to_mastodon.sh b/.github/workflows/post_to_mastodon.sh index 20a65a23..045e87f6 100644 --- a/.github/workflows/post_to_mastodon.sh +++ b/.github/workflows/post_to_mastodon.sh @@ -2,7 +2,7 @@ # Extract version from PR tag passed as environment variable if [ -z "${PR_TITLE}" ]; then # apparently unset, workflow broken? - echo "Error: 'PR_TITLE' environment variable is not set." + >&2 echo "Error: 'PR_TITLE' environment variable is not set." exit 1 fi version="${PR_TITLE##* }" From e0059c850a219896eec26253117a7cf1f5996279 Mon Sep 17 00:00:00 2001 From: Christian Meesters Date: Tue, 11 Mar 2025 16:49:11 +0100 Subject: [PATCH 2/6] fix: allow an account to be explicitly NOT set --- snakemake_executor_plugin_slurm/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/snakemake_executor_plugin_slurm/__init__.py b/snakemake_executor_plugin_slurm/__init__.py index a2a29150..f168a518 100644 --- a/snakemake_executor_plugin_slurm/__init__.py +++ b/snakemake_executor_plugin_slurm/__init__.py @@ -85,6 +85,15 @@ class ExecutorSettings(ExecutorSettingsBase): "required": False, }, ) + no_account: bool = field( + default=False, + metadata={ + "help": "Do not use any account for submission. " + "This flag has no effect, if not set.", + "env_var": False, + "required": False, + }, + ) # Required: @@ -213,7 +222,9 @@ def run_job(self, job: JobExecutorInterface): f"--comment '{comment_str}'" ) - call += self.get_account_arg(job) + if not self.workflow.executor_settings.no_account: + call += self.get_account_arg(job) + call += self.get_partition_arg(job) if self.workflow.executor_settings.requeue: From 29ab5e9db4e0eb36d2c072f4b53bbbe5080c5829 Mon Sep 17 00:00:00 2001 From: Christian Meesters Date: Wed, 12 Mar 2025 14:57:03 +0100 Subject: [PATCH 3/6] docs: added rudimentary documentation about the new --slurm-no-account flag --- docs/further.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/further.md b/docs/further.md index 29c22f0e..b56ef1ed 100644 --- a/docs/further.md +++ b/docs/further.md @@ -18,6 +18,9 @@ To specify them at the command line, define them as default resources: $ snakemake --executor slurm --default-resources slurm_account= slurm_partition= ``` +The plugin does its best to _guess_ your account. That might not be possible. Particularly, when dealing with several SLURM accounts, users ought to set them per workflow. +Some clusters, however, have a pre-defined default per user and _do not_ allow users to set their account. The plugin will attept to always set an account. To override this behauvior, the `--slurm-no-account` flag can be used. + If individual rules require e.g. a different partition, you can override the default per rule: From f1e0abecde58a5a8941d2fc512b33b783036c347 Mon Sep 17 00:00:00 2001 From: Christian Meesters Date: Wed, 12 Mar 2025 22:46:02 +0100 Subject: [PATCH 4/6] fix: reshuffled command order and made more stable testing for account validity --- snakemake_executor_plugin_slurm/__init__.py | 32 ++++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/snakemake_executor_plugin_slurm/__init__.py b/snakemake_executor_plugin_slurm/__init__.py index f168a518..6f3d1ceb 100644 --- a/snakemake_executor_plugin_slurm/__init__.py +++ b/snakemake_executor_plugin_slurm/__init__.py @@ -645,35 +645,45 @@ def test_account(self, account): """ tests whether the given account is registered, raises an error, if not """ - cmd = f'sacctmgr -n -s list user "{os.environ["USER"]}" format=account%256' + cmd = "sshare -U --format Account --noheader" try: accounts = subprocess.check_output( cmd, shell=True, text=True, stderr=subprocess.PIPE ) except subprocess.CalledProcessError as e: - sacctmgr_report = ( - "Unable to test the validity of the given or guessed " - f"SLURM account '{account}' with sacctmgr: {e.stderr}." + sshare_report = ( + "Unable to test the validity of the given or guessed" + f" SLURM account '{account}' with sshare: {e.stderr}." ) + accounts = "" + + if not accounts.strip(): + cmd = f'sacctmgr -n -s list user "{os.environ["USER"]}" format=account%256' try: - cmd = "sshare -U --format Account --noheader" accounts = subprocess.check_output( cmd, shell=True, text=True, stderr=subprocess.PIPE ) - except subprocess.CalledProcessError as e2: - sshare_report = ( - "Unable to test the validity of the given or guessed" - f" SLURM account '{account}' with sshare: {e2.stderr}." + except subprocess.CalledProcessError as e: + sacctmgr_report = ( + "Unable to test the validity of the given or guessed " + f"SLURM account '{account}' with sacctmgr: {e.stderr}." ) raise WorkflowError( - f"The 'sacctmgr' reported: '{sacctmgr_report}' " - f"and likewise 'sshare' reported: '{sshare_report}'." + f"The 'sshare' reported: '{sshare_report}' " + f"and likewise 'sacctmgr' reported: '{sacctmgr_report}'." ) # The set() has been introduced during review to eliminate # duplicates. They are not harmful, but disturbing to read. accounts = set(_.strip() for _ in accounts.split("\n") if _) + if not accounts: + self.logger.warning( + f"Both 'sshare' and 'sacctmgr' returned empty results for account " + f"'{account}'. Proceeding without account validation." + ) + return "" + if account not in accounts: raise WorkflowError( f"The given account {account} appears to be invalid. Available " From 4f4945b80f2121bf79092c7883e155c74677007e Mon Sep 17 00:00:00 2001 From: Christian Meesters Date: Wed, 12 Mar 2025 22:50:47 +0100 Subject: [PATCH 5/6] fix: solved merge confligt by deleting offending post_to_mastodon shell script --- .github/workflows/post_to_mastodon.sh | 79 --------------------------- 1 file changed, 79 deletions(-) delete mode 100644 .github/workflows/post_to_mastodon.sh diff --git a/.github/workflows/post_to_mastodon.sh b/.github/workflows/post_to_mastodon.sh deleted file mode 100644 index 98683750..00000000 --- a/.github/workflows/post_to_mastodon.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -# Extract version from PR tag passed as environment variable -if [ -z "${PR_TITLE}" ]; then # apparently unset, workflow broken? - >&2 echo "Error: 'PR_TITLE' environment variable is not set." - exit 1 -fi - -# Check if this is a release PR -if [[ ! "${PR_TITLE}" =~ ^chore\(main\):\ release ]]; then - echo "Not a release PR, skipping Mastodon post" - exit 0 -fi - -# Extract version (everything after "release ") -if [[ "${PR_TITLE}" =~ [Rr]elease[[:space:]]+([0-9]+\.[0-9]+\.[0-9]+) ]]; then - version="${BASH_REMATCH[1]}" -else - echo "Error: Could not extract version number from PR title" - exit 1 -fi - -# Validate version format -if ! [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "Error: Invalid version format in PR title: $version" - exit 1 -fi - -# Construct changelog URL with proper quoting -changelog="https://github.com/snakemake/snakemake-executor-plugin-slurm/releases/tag/v${version}" - -# Maximum character limit for Mastodon posts (on Fediscience: 1500 characters) -MAX_TOOT_LENGTH=1500 - -read -d '\n' message << EndOfText -Beep, Beep - I am your friendly #Snakemake release announcement bot. - -There is a new release of the Snakemake executor for #SLURM on #HPC systems. Its version is '${version}'! - -See ${changelog//\'/\\\'} for details. - -Give us some time and you will automatically find the plugin on #Bioconda and #Pypi. - -If you want to discuss the release you will find the maintainers here on Mastodon! -@rupdecat@fediscience.org and @johanneskoester@fosstodon.org - -If you find any issues, please report them on https://github.com/snakemake/snakemake-executor-plugin-slurm/issues - -#Snakemake #HPC #ReproducibleComputing #ReproducibleResearch #OpenScience -EndOfText - -# Validate message length -if [ ${#message} -gt $MAX_TOOT_LENGTH ]; then - echo "Error: Message exceeds Fediscience's character limit" - exit 1 -fi - -# Validate Mastodon token -if [ -z "${MASTODONBOT}" ]; then - echo "Error: MASTODONBOT secret is not set" - exit 1 -fi - -# Send post to Mastodon with proper quoting and error handling -response=$(curl -s -w "\n%{http_code}" -X POST \ - -H "Authorization: Bearer ${MASTODONBOT}" \ - -F "status=${message}" \ - "https://fediscience.org/api/v1/statuses") - -status_code=$(echo "$response" | tail -n1) -response_body=$(echo "$response" | sed '$d') - -if [ "$status_code" -ne 200 ]; then - echo "Error: Failed to post to Mastodon (HTTP ${status_code})" - echo "Response: ${response_body}" - exit 1 -fi - -echo "Successfully posted to Mastodon" From 7e8318fc7e9ce3e68420a79c417abf6357029eda Mon Sep 17 00:00:00 2001 From: DrYak Date: Thu, 13 Mar 2025 11:28:48 +0100 Subject: [PATCH 6/6] Doc: further.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - correct typos and grammar, based on languagetool.org and coderabbitai - mention partitions as another thing that can be blocked by local cluster settings (e.g.: on ETH Zürich's Euler) --- docs/further.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/further.md b/docs/further.md index b56ef1ed..03205000 100644 --- a/docs/further.md +++ b/docs/further.md @@ -19,7 +19,7 @@ $ snakemake --executor slurm --default-resources slurm_account=