diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml
index 41e6820dd8..85e9198855 100644
--- a/.github/workflows/container_tests.yml
+++ b/.github/workflows/container_tests.yml
@@ -73,7 +73,7 @@ jobs:
python setup.py sdist
ls dist
export PREFIX=/tmp/$USER/$GITHUB_SHA
- pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+ pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz
- name: run test
diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml
index 1539221489..4f9df0df9a 100644
--- a/.github/workflows/container_tests_apptainer.yml
+++ b/.github/workflows/container_tests_apptainer.yml
@@ -73,7 +73,7 @@ jobs:
python setup.py sdist
ls dist
export PREFIX=/tmp/$USER/$GITHUB_SHA
- pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+ pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz
- name: run test
diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml
index a41b50f572..625206b870 100644
--- a/.github/workflows/eb_command.yml
+++ b/.github/workflows/eb_command.yml
@@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
- python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+ python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12', '3.13']
fail-fast: false
steps:
- uses: actions/checkout@v3
@@ -32,6 +32,10 @@ jobs:
# update to latest pip, check version
pip install --upgrade pip
pip --version
+ if ! python -c "import distutils" 2> /dev/null; then
+ # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist
+ pip install --upgrade setuptools
+ fi
# for modules tool
APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
@@ -68,7 +72,7 @@ jobs:
python setup.py sdist
ls dist
export PREFIX=/tmp/$USER/$GITHUB_SHA
- pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+ pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
- name: run tests for 'eb' command
env:
diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
index fc4936286a..ed6098de60 100644
--- a/.github/workflows/end2end.yml
+++ b/.github/workflows/end2end.yml
@@ -17,6 +17,7 @@ jobs:
fail-fast: false
container:
image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+ env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions
steps:
- name: Check out the repo
uses: actions/checkout@v3
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
index 93af5bfc29..9dba1ed474 100644
--- a/.github/workflows/linting.yml
+++ b/.github/workflows/linting.yml
@@ -13,8 +13,7 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
- python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
-
+ python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12', '3.13']
steps:
- uses: actions/checkout@v3
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
index 6a8df107d3..e7b5e2b50d 100644
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -14,7 +14,8 @@ jobs:
runs-on: ubuntu-20.04
outputs:
lmod8: Lmod-8.7.6
- modules4: modules-4.1.4
+ modules4: modules-4.5.3
+ modules5: modules-5.3.1
steps:
- run: "true"
build:
@@ -28,6 +29,7 @@ jobs:
# https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
- ${{needs.setup.outputs.lmod8}}
- ${{needs.setup.outputs.modules4}}
+ - ${{needs.setup.outputs.modules5}}
lc_all: [""]
include:
# Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
@@ -41,6 +43,10 @@ jobs:
modules_tool: ${{needs.setup.outputs.lmod8}}
- python: '3.11'
modules_tool: ${{needs.setup.outputs.lmod8}}
+ - python: '3.12'
+ modules_tool: ${{needs.setup.outputs.lmod8}}
+ - python: '3.13'
+ modules_tool: ${{needs.setup.outputs.lmod8}}
# There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
# Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
- python: 3.6
@@ -83,6 +89,10 @@ jobs:
pip install --upgrade pip
pip --version
pip install -r requirements.txt
+ if ! python -c "import distutils" 2> /dev/null; then
+ # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist
+ pip install --upgrade setuptools
+ fi
# git config is required to make actual git commits (cfr. tests for GitRepository)
git config --global user.name "Travis CI"
git config --global user.email "travis@travis-ci.org"
@@ -95,12 +105,12 @@ jobs:
# and are only run after the PR gets merged
GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
run: |
- # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit;
+ # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit
# tests that require a GitHub token are skipped automatically when no GitHub token is available
if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
if [ ! -z $GITHUB_TOKEN ]; then
- SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
- python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+ SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"
+ python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"
fi
echo "GitHub token installed!"
else
@@ -132,7 +142,7 @@ jobs:
python setup.py sdist
ls dist
export PREFIX=/tmp/$USER/$GITHUB_SHA
- pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+ pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
- name: run test suite
env:
@@ -144,7 +154,9 @@ jobs:
cd $HOME
# initialize environment for modules tool
if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
- source $(cat $HOME/mod_init); type module
+ source $(cat $HOME/mod_init)
+ type module
+ module --version
# make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
# also pick up changes to $PATH set by sourcing $MOD_INIT
export PREFIX=/tmp/$USER/$GITHUB_SHA
@@ -152,7 +164,7 @@ jobs:
export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
eb --version
# tell EasyBuild which modules tool is available
- if [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
+ if [[ ${{matrix.modules_tool}} =~ ^modules- ]]; then
export EASYBUILD_MODULES_TOOL=EnvironmentModules
else
export EASYBUILD_MODULES_TOOL=Lmod
@@ -181,7 +193,17 @@ jobs:
# run test suite
python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
# try and make sure output of running tests is clean (no printed messages/warnings)
- IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
+ IGNORE_PATTERNS="no GitHub token available"
+ IGNORE_PATTERNS+="|skipping SvnRepository test"
+ IGNORE_PATTERNS+="|requires Lmod as modules tool"
+ IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device"
+ IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]"
+ IGNORE_PATTERNS+="|from cryptography.* import "
+ IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2"
+ IGNORE_PATTERNS+="|Blowfish"
+ IGNORE_PATTERNS+="|GC3Pie not available, skipping test"
+ IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved"
+ IGNORE_PATTERNS+="|algorithms.TripleDES"
# '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 94b66db903..d531df52d7 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -4,6 +4,112 @@ For more detailed information, please see the git log.
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
+v4.9.4 (22 September 2024)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+ - set $LMOD_TERSE_DECORATIONS to 'no' to avoid additional info in output produced by 'ml --terse avail' (#4648)
+- various bug fixes, including:
+ - implement workaround for permission error when copying read-only files that have extended attributes set and using Python 3.6 (#4642)
+ - take into account alternate sysroot for /bin/bash used by run_cmd (#4646)
+
+
+v4.9.3 (14 September 2024)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+ - add support for `--extra-source-urls` to fetch sources from additional URLs (#4079)
+ - add definition for gmpflf toolchain (#4566, #4571)
+ - reuse pre-computed checksums (#4569)
+ - add `cuda_cc_space_sep` variant that does not have periods (#4583)
+ - add `--skip-sanity-check` option (#4590)
+ - add `GNU_FTP_SOURCE` template constant (#4597)
+ - improve error messages for empty easyconfigs (#4603)
+ - improve help string for `--dep-graph` (#4610)
+ - only call `_sanity_check_step_extensions` if `--skip-extensions` is not set (#4620)
+ - add support for `--software-commit` and an associated template `%(software_commit)s` (#4628)
+- various bug fixes, including:
+ - correctly evaluate result for `--dep-graph` (#4554)
+ - fix fetch progress bar showing to many files (#4568)
+ - resolve internal for imkl>=2021 version subdir via "latest" symlink (#4570)
+ - fix typo in message about including an easyblock from a commit (#4575)
+ - don't use special flags for `strict`, `precise`, `loose`, `veryloose` toolchain options on RISC-V (#4576)
+ - fix help text for `cuda_compute_capabilities` template (#4589)
+ - fix help message for `--http-headers-fields-urlpat` configuration option (#4594)
+ - fix `test_compiler_cache` in case `gcc` is available multiple times (#4599)
+ - handle post-install patches in check_checksums_for (#4605)
+ - fix `copy_file` with a folder as the target (#4609)
+ - allow for case where `homepage = None` when generating the docs (#4626)
+ - fix test_github_det_commit_status by using more recent commits (#4636)
+- other changes:
+ - clean up code that was only there to support Python 2.6 + avoid syntax warnings when parsing py2vs3/py.p2 with Python 3.x (#3788)
+ - use Intel's oneAPI Fortran compiler by default for version 2024.0.0 and newer (`oneapi_fortran` toolchain option set to `True`) (#4567)
+ - allow using Node 16 actions in CI (#4574)
+ - remove a superflous check in `EasyBlock.run_all_steps` (#4623)
+ - remove trailing dots from backup message produced by --inject-checksums (#4632)
+
+
+v4.9.2 (12 June 2024)
+---------------------
+
+update/bugfix release
+
+- various enhancements, including:
+ - improve behavior when using extension which has 'nosource' enabled (#4506)
+ - enhance 'get_software_libdir' to return 'lib' or 'lib64' if only one of them contains library files (#4513)
+ - implement versions checks to avoid mixing major versions across the EasyBuild components (#4520, #4553)
+ - add support for easyconfig parameter 'module_only' (#4537)
+- various bug fixes, including:
+ - fix typo in patch_step logging (#4505)
+ - consider both 'easybuild-framework*.tar.gz' and 'easybuild_framework*.tar.gz' in CI workflows (#4507)
+ - don't delete existing environment module files when using '--dump-env-script' with '--force' or '--rebuild' (#4512)
+ - fix resolved (template) values in case of failure (#4532)
+ - also consider '$CRAY_PE_LIBSCI_PREFIX_DIR' to determine installation prefix for cray-libsci (#4551)
+ - symlink downloaded repo at specified commit when using '--from-commit' so easyconfigs for dependencies are found (#4552)
+- other changes:
+ - code cleanup in 'easyblock.py' (#4519)
+ - stop running unit tests on Python 3.5 (#4530)
+
+
+v4.9.1 (5 April 2024)
+---------------------
+
+update/bugfix release
+
+- various enhancements, including:
+ - make `is_rpath_wrapper` faster by only checking file contents if file is not located in subdirectory of RPATH wrapper subdirectory (#4406)
+ - add terse support to `--missing-modules` (#4407)
+ - adapt version pattern for EnvironmentModules to allow using development version (#4416)
+ - use `--all` option with EnvironmentModules v4.6+ to get available hidden modules (#4417)
+ - add support for appending to path environment variables via `modextrapaths_append` + add corresponding `allow_append_abs_path` (#4436)
+ - improve output produced by `--check-github` (#4437)
+ - add script for updating local git repos with `develop` branch (#4438)
+ - show error when multiple PR options are passed (#4440)
+ - improve `findPythonDeps` script to recognize non-canonical package names (#4445)
+ - add support for `--from-commit` and `--include-easyblocks-from-commit` (#4468)
+ - improve logging & handling of (empty) `--optarch` values (#4481)
+ - add `--short` option to `findUpdatedEcs` script (#4488)
+ - add generic GCC and Clang compiler flags for RISC-V (#4489)
+- various bug fixes, including:
+ - clean up log file of `EasyBlock` instance in `check_sha256_checksums` (#4452)
+ - fix description of `backup-modules` configuration option (#4456)
+ - replace `'` with `"` for `printf` in CI workflow for running test suite to have bash replace a variable (#4461)
+ - use `cp -dR` instead of `cp -a` for shell script "extraction" (#4465)
+ - fix link to documentation in `close_pr` message (#4466)
+ - fix `test_github_merge_pr` by using more recent easyconfigs PR (#4470)
+ - add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration (#4472)
+- other changes:
+ - clean up & speed up environment checks (#4409)
+ - use more performant and concise dict construction by using dict comprehensions (#4410)
+ - remove superflous string formatting (#4411)
+ - clean up uses of `getattr` and `hasattr` (#4412)
+ - update copyright lines to 2024 (#4494)
+
+
v4.9.0 (30 December 2023)
-------------------------
diff --git a/easybuild/__init__.py b/easybuild/__init__.py
index 4763af8bc4..3637c9f395 100644
--- a/easybuild/__init__.py
+++ b/easybuild/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/_deprecated.py b/easybuild/_deprecated.py
new file mode 100644
index 0000000000..2beebd8174
--- /dev/null
+++ b/easybuild/_deprecated.py
@@ -0,0 +1,854 @@
+# #
+# Copyright 2023-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+# #
+"""
+Deprecated functionality, which will be removed with next major EasyBuild version
+
+Authors:
+
+* Kenneth Hoste (Ghent University)
+"""
+import contextlib
+import functools
+import os
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+from datetime import datetime
+
+import easybuild.tools.asyncprocess as asyncprocess
+from easybuild.base import fancylogger
+from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since
+from easybuild.tools.config import ERROR, IGNORE, WARN, build_option
+from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook
+from easybuild.tools.utilities import nub, trace_msg
+
+
+_log = fancylogger.getLogger('_deprecated', fname=False)
+
+
+errors_found_in_log = 0
+
+# default strictness level
+strictness = WARN
+
+
+CACHED_COMMANDS = [
+ "sysctl -n hw.cpufrequency_max", # used in get_cpu_speed (OS X)
+ "sysctl -n hw.memsize", # used in get_total_memory (OS X)
+ "sysctl -n hw.ncpu", # used in get_avail_core_count (OS X)
+ "sysctl -n machdep.cpu.brand_string", # used in get_cpu_model (OS X)
+ "sysctl -n machdep.cpu.vendor", # used in get_cpu_vendor (OS X)
+ "type module", # used in ModulesTool.check_module_function
+ "type _module_raw", # used in EnvironmentModules.check_module_function
+ "ulimit -u", # used in det_parallelism
+]
+
+
+def run_cmd_cache(func):
+ """Function decorator to cache (and retrieve cached) results of running commands."""
+ cache = {}
+
+ @functools.wraps(func)
+ def cache_aware_func(cmd, *args, **kwargs):
+ """Retrieve cached result of selected commands, or run specified and collect & cache result."""
+
+ # cache key is combination of command and input provided via stdin ('inp' named option)
+ key = (cmd, kwargs.get('inp', None))
+ # fetch from cache if available, cache it if it's not, but only on cmd strings
+ if isinstance(cmd, str) and key in cache:
+ _log.debug("Using cached value for command '%s': %s", cmd, cache[key])
+ return cache[key]
+ else:
+ res = func(cmd, *args, **kwargs)
+ if cmd in CACHED_COMMANDS:
+ cache[key] = res
+ return res
+
+ # expose clear/update methods of cache to wrapped function
+ cache_aware_func.clear_cache = cache.clear
+ cache_aware_func.update_cache = cache.update
+
+ return cache_aware_func
+
+
+def get_output_from_process(proc, read_size=None, asynchronous=False, print_deprecation_warning=True):
+ """
+ Get output from running process (that was opened with subprocess.Popen).
+
+ :param proc: process to get output from
+ :param read_size: number of bytes of output to read (if None: read all output)
+ :param asynchronous: get output asynchronously
+ """
+
+ if print_deprecation_warning:
+ _log.deprecated("get_output_from_process is deprecated, you should stop using it", '6.0')
+
+ if asynchronous:
+ # e=False is set to avoid raising an exception when command has completed;
+ # that's needed to ensure we get all output,
+ # see https://github.com/easybuilders/easybuild-framework/issues/3593
+ output = asyncprocess.recv_some(proc, e=False)
+ elif read_size:
+ output = proc.stdout.read(read_size)
+ else:
+ output = proc.stdout.read()
+
+ # need to be careful w.r.t. encoding since we want to obtain a string value,
+ # and the output may include non UTF-8 characters
+ # * in Python 2, .decode() returns a value of type 'unicode',
+ # but we really want a regular 'str' value (which is also why we use 'ignore' for encoding errors)
+ # * in Python 3, .decode() returns a 'str' value when called on the 'bytes' value obtained from .read()
+ output = str(output.decode('ascii', 'ignore'))
+
+ return output
+
+
+@run_cmd_cache
+def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None,
+ force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False,
+ with_hooks=True, with_sysroot=True):
+ """
+ Run specified command (in a subshell)
+ :param cmd: command to run
+ :param log_ok: only run output/exit code for failing commands (exit code non-zero)
+ :param log_all: always log command output and exit code
+ :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
+ :param inp: the input given to the command via stdin
+ :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
+ :param log_output: indicate whether all output of command should be logged to a separate temporary logfile
+ :param path: path to execute the command in; current working directory is used if unspecified
+ :param force_in_dry_run: force running the command during dry run
+ :param verbose: include message on running the command in dry run output
+ :param shell: allow commands to not run in a shell (especially useful for cmd lists), defaults to True
+ :param trace: print command being executed as part of trace output
+ :param stream_output: enable streaming command output to stdout
+ :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True)
+ :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined)
+ :param with_sysroot: prepend sysroot to exec_cmd (if defined)
+ """
+
+ _log.deprecated("run_cmd is deprecated, use run_shell_cmd from easybuild.tools.run instead", '6.0')
+
+ cwd = os.getcwd()
+
+ if isinstance(cmd, str):
+ cmd_msg = cmd.strip()
+ elif isinstance(cmd, list):
+ cmd_msg = ' '.join(cmd)
+ else:
+ raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
+
+ if shell is None:
+ shell = True
+ if isinstance(cmd, list):
+ raise EasyBuildError("When passing cmd as a list then `shell` must be set explictely! "
+ "Note that all elements of the list but the first are treated as arguments "
+ "to the shell and NOT to the command to be executed!")
+
+ if log_output or (trace and build_option('trace')):
+ # collect output of running command in temporary log file, if desired
+ fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-')
+ os.close(fd)
+ try:
+ cmd_log = open(cmd_log_fn, 'w')
+ except IOError as err:
+ raise EasyBuildError("Failed to open temporary log file for output of command: %s", err)
+ _log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
+ else:
+ cmd_log_fn, cmd_log = None, None
+
+ # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
+ if stream_output is None and build_option('logtostdout'):
+ _log.info("Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg)
+ stream_output = True
+
+ if stream_output:
+ print_msg("(streaming) output for command '%s':" % cmd_msg)
+
+ start_time = datetime.now()
+ if trace:
+ trace_txt = "running command:\n"
+ trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
+ trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
+ if inp:
+ trace_txt += "\t[input: %s]\n" % inp
+ trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
+ trace_msg(trace_txt + '\t' + cmd_msg)
+
+ # early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
+ if not force_in_dry_run and build_option('extended_dry_run'):
+ if path is None:
+ path = cwd
+ if verbose:
+ dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent'))
+ dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
+
+ # make sure we get the type of the return value right
+ if simple:
+ return True
+ else:
+ # output, exit code
+ return ('', 0)
+
+ try:
+ if path:
+ os.chdir(path)
+
+ _log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
+ except OSError as err:
+ _log.warning("Failed to change to %s: %s" % (path, err))
+ _log.info("running cmd %s in non-existing directory, might fail!", cmd)
+
+ if cmd_log:
+ cmd_log.write("# output for command: %s\n\n" % cmd_msg)
+
+ exec_cmd = "/bin/bash"
+
+ # if EasyBuild is configured to use an alternate sysroot,
+ # we should also run shell commands using the bash shell provided in there,
+ # since /bin/bash may not be compatible with the alternate sysroot
+ if with_sysroot:
+ sysroot = build_option('sysroot')
+ if sysroot:
+ sysroot_bin_bash = os.path.join(sysroot, 'bin', 'bash')
+ if os.path.exists(sysroot_bin_bash):
+ exec_cmd = sysroot_bin_bash
+
+ if not shell:
+ if isinstance(cmd, list):
+ exec_cmd = None
+ cmd.insert(0, '/usr/bin/env')
+ elif isinstance(cmd, str):
+ cmd = '/usr/bin/env %s' % cmd
+ else:
+ raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd))
+
+ _log.info("Using %s as shell for running cmd: %s", exec_cmd, cmd)
+
+ if with_hooks:
+ hooks = load_hooks(build_option('hooks'))
+ hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()})
+ if isinstance(hook_res, str):
+ cmd, old_cmd = hook_res, cmd
+ _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd)
+
+ _log.info('running cmd: %s ' % cmd)
+ try:
+ proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd)
+ except OSError as err:
+ raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err)
+
+ if inp:
+ proc.stdin.write(inp.encode())
+ proc.stdin.close()
+
+ if asynchronous:
+ return (proc, cmd, cwd, start_time, cmd_log)
+ else:
+ return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple,
+ regexp=regexp, stream_output=stream_output, trace=trace, with_hook=with_hooks,
+ print_deprecation_warning=False)
+
+
+def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''):
+ """
+ Check status of command that was started asynchronously.
+
+ :param proc: subprocess.Popen instance representing asynchronous command
+ :param cmd: command being run
+ :param owd: original working directory
+ :param start_time: start time of command (datetime instance)
+ :param cmd_log: log file to print command output to
+ :param fail_on_error: raise EasyBuildError when command exited with an error
+ :param output_read_size: number of bytes to read from output
+ :param output: already collected output for this command
+
+ :result: dict value with result of the check (boolean 'done', 'exit_code', 'output')
+ """
+
+ _log.deprecated("check_async_cmd is deprecated, you should stop using it", '6.0')
+
+ # use small read size, to avoid waiting for a long time until sufficient output is produced
+ if output_read_size:
+ if not isinstance(output_read_size, int) or output_read_size < 0:
+ raise EasyBuildError("Number of output bytes to read should be a positive integer value (or zero)")
+ add_out = get_output_from_process(proc, read_size=output_read_size, print_deprecation_warning=False)
+ _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out))
+ output += add_out
+
+ exit_code = proc.poll()
+ if exit_code is None:
+ _log.debug("Asynchronous command '%s' still running..." % cmd)
+ done = False
+ else:
+ _log.debug("Asynchronous command '%s' completed!", cmd)
+ output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output,
+ simple=False, trace=False, log_ok=fail_on_error,
+ print_deprecation_warning=False)
+ done = True
+
+ res = {
+ 'done': done,
+ 'exit_code': exit_code,
+ 'output': output,
+ }
+ return res
+
+
+def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False,
+ regexp=True, stream_output=None, trace=True, output='', with_hook=True,
+ print_deprecation_warning=True):
+ """
+ Complete running of command represented by passed subprocess.Popen instance.
+
+ :param proc: subprocess.Popen instance representing running command
+ :param cmd: command being run
+ :param owd: original working directory
+ :param start_time: start time of command (datetime instance)
+ :param cmd_log: log file to print command output to
+ :param log_ok: only run output/exit code for failing commands (exit code non-zero)
+ :param log_all: always log command output and exit code
+ :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
+ :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
+ :param stream_output: enable streaming command output to stdout
+ :param trace: print command being executed as part of trace output
+ :param with_hook: trigger post run_shell_cmd hooks (if defined)
+ """
+
+ if print_deprecation_warning:
+ _log.deprecated("complete_cmd is deprecated, you should stop using it", '6.0')
+
+ # use small read size when streaming output, to make it stream more fluently
+ # read size should not be too small though, to avoid too much overhead
+ if stream_output:
+ read_size = 128
+ else:
+ read_size = 1024 * 8
+
+ stdouterr = output
+
+ try:
+ ec = proc.poll()
+ while ec is None:
+ # need to read from time to time.
+ # - otherwise the stdout/stderr buffer gets filled and it all stops working
+ output = get_output_from_process(proc, read_size=read_size, print_deprecation_warning=False)
+ if cmd_log:
+ cmd_log.write(output)
+ if stream_output:
+ sys.stdout.write(output)
+ stdouterr += output
+ ec = proc.poll()
+
+ # read remaining data (all of it)
+ output = get_output_from_process(proc, print_deprecation_warning=False)
+ finally:
+ proc.stdout.close()
+
+ if cmd_log:
+ cmd_log.write(output)
+ cmd_log.close()
+ if stream_output:
+ sys.stdout.write(output)
+ stdouterr += output
+
+ if with_hook:
+ hooks = load_hooks(build_option('hooks'))
+ run_hook_kwargs = {
+ 'exit_code': ec,
+ 'output': stdouterr,
+ 'work_dir': os.getcwd(),
+ }
+ run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
+
+ if trace:
+ trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
+
+ try:
+ os.chdir(owd)
+ except OSError as err:
+ raise EasyBuildError("Failed to return to %s after executing command: %s", owd, err)
+
+ return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp, print_deprecation_warning=False)
+
+
+def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None,
+ maxhits=50, trace=True):
+ """
+ Run specified interactive command (in a subshell)
+ :param cmd: command to run
+ :param qa: dictionary which maps question to answers
+ :param no_qa: list of patters that are not questions
+ :param log_ok: only run output/exit code for failing commands (exit code non-zero)
+ :param log_all: always log command output and exit code
+ :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
+ :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
+ :param std_qa: dictionary which maps question regex patterns to answers
+ :param path: path to execute the command is; current working directory is used if unspecified
+ :param maxhits: maximum number of cycles (seconds) without being able to find a known question
+ :param trace: print command being executed as part of trace output
+ """
+
+ _log.deprecated("run_cmd_qa is deprecated, use run_shell_cmd from easybuild.tools.run instead", '6.0')
+
+ cwd = os.getcwd()
+
+ if not isinstance(cmd, str) and len(cmd) > 1:
+ # We use shell=True and hence we should really pass the command as a string
+ # When using a list then every element past the first is passed to the shell itself, not the command!
+ raise EasyBuildError("The command passed must be a string!")
+
+ if log_all or (trace and build_option('trace')):
+ # collect output of running command in temporary log file, if desired
+ fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-')
+ os.close(fd)
+ try:
+ cmd_log = open(cmd_log_fn, 'w')
+ except IOError as err:
+ raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err)
+ _log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
+ else:
+ cmd_log_fn, cmd_log = None, None
+
+ start_time = datetime.now()
+ if trace:
+ trace_txt = "running interactive command:\n"
+ trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
+ trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
+ trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
+ trace_msg(trace_txt + '\t' + cmd.strip())
+
+ # early exit in 'dry run' mode, after printing the command that would be run
+ if build_option('extended_dry_run'):
+ if path is None:
+ path = cwd
+ dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent'))
+ dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
+ if cmd_log:
+ cmd_log.close()
+ if simple:
+ return True
+ else:
+ # output, exit code
+ return ('', 0)
+
+ try:
+ if path:
+ os.chdir(path)
+
+ _log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
+ except OSError as err:
+ _log.warning("Failed to change to %s: %s" % (path, err))
+ _log.info("running cmd %s in non-existing directory, might fail!" % cmd)
+
+ # Part 1: process the QandA dictionary
+ # given initial set of Q and A (in dict), return dict of reg. exp. and A
+ #
+ # make regular expression that matches the string with
+ # - replace whitespace
+ # - replace newline
+
+ def escape_special(string):
+ return re.sub(r"([\+\?\(\)\[\]\*\.\\\$])", r"\\\1", string)
+
+ split = r'[\s\n]+'
+ regSplit = re.compile(r"" + split)
+
+ def process_QA(q, a_s):
+ splitq = [escape_special(x) for x in regSplit.split(q)]
+ regQtxt = split.join(splitq) + split.rstrip('+') + "*$"
+ # add optional split at the end
+ for i in [idx for idx, a in enumerate(a_s) if not a.endswith('\n')]:
+ a_s[i] += '\n'
+ regQ = re.compile(r"" + regQtxt)
+ if regQ.search(q):
+ return (a_s, regQ)
+ else:
+ raise EasyBuildError("runqanda: Question %s converted in %s does not match itself", q, regQtxt)
+
+ def check_answers_list(answers):
+ """Make sure we have a list of answers (as strings)."""
+ if isinstance(answers, str):
+ answers = [answers]
+ elif not isinstance(answers, list):
+ if cmd_log:
+ cmd_log.close()
+ raise EasyBuildError("Invalid type for answer on %s, no string or list: %s (%s)",
+ question, type(answers), answers)
+ # list is manipulated when answering matching question, so return a copy
+ return answers[:]
+
+ new_qa = {}
+ _log.debug("new_qa: ")
+ for question, answers in qa.items():
+ answers = check_answers_list(answers)
+ (answers, regQ) = process_QA(question, answers)
+ new_qa[regQ] = answers
+ _log.debug("new_qa[%s]: %s" % (regQ.pattern, new_qa[regQ]))
+
+ new_std_qa = {}
+ if std_qa:
+ for question, answers in std_qa.items():
+ regQ = re.compile(r"" + question + r"[\s\n]*$")
+ answers = check_answers_list(answers)
+ for i in [idx for idx, a in enumerate(answers) if not a.endswith('\n')]:
+ answers[i] += '\n'
+ new_std_qa[regQ] = answers
+ _log.debug("new_std_qa[%s]: %s" % (regQ.pattern, new_std_qa[regQ]))
+
+ new_no_qa = []
+ if no_qa:
+ # simple statements, can contain wildcards
+ new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa]
+
+ _log.debug("New noQandA list is: %s" % [x.pattern for x in new_no_qa])
+
+ # Part 2: Run the command and answer questions
+ # - this needs asynchronous stdout
+
+ hooks = load_hooks(build_option('hooks'))
+ run_hook_kwargs = {
+ 'interactive': True,
+ 'work_dir': os.getcwd(),
+ }
+ hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
+ if isinstance(hook_res, str):
+ cmd, old_cmd = hook_res, cmd
+ _log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')",
+ RUN_SHELL_CMD, cmd, old_cmd)
+
+ # # Log command output
+ if cmd_log:
+ cmd_log.write("# output for interactive command: %s\n\n" % cmd)
+
+ # Make sure we close the proc handles and the cmd_log file
+ @contextlib.contextmanager
+ def get_proc():
+ try:
+ proc = asyncprocess.Popen(cmd, shell=True, stdout=asyncprocess.PIPE, stderr=asyncprocess.STDOUT,
+ stdin=asyncprocess.PIPE, close_fds=True, executable='/bin/bash')
+ except OSError as err:
+ if cmd_log:
+ cmd_log.close()
+ raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err)
+ try:
+ yield proc
+ finally:
+ if proc.stdout:
+ proc.stdout.close()
+ if proc.stdin:
+ proc.stdin.close()
+ if cmd_log:
+ cmd_log.close()
+
+ with get_proc() as proc:
+ ec = proc.poll()
+ stdout_err = ''
+ old_len_out = -1
+ hit_count = 0
+
+ while ec is None:
+ # need to read from time to time.
+ # - otherwise the stdout/stderr buffer gets filled and it all stops working
+ try:
+ out = get_output_from_process(proc, asynchronous=True, print_deprecation_warning=False)
+
+ if cmd_log:
+ cmd_log.write(out)
+ stdout_err += out
+ # recv_some used by get_output_from_process for getting asynchronous output may throw exception
+ except (IOError, Exception) as err:
+ _log.debug("run_cmd_qa cmd %s: read failed: %s", cmd, err)
+ out = None
+
+ hit = False
+ for question, answers in new_qa.items():
+ res = question.search(stdout_err)
+ if out and res:
+ fa = answers[0] % res.groupdict()
+ # cycle through list of answers
+ last_answer = answers.pop(0)
+ answers.append(last_answer)
+ _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers)
+
+ _log.debug("run_cmd_qa answer %s question %s out %s", fa, question.pattern, stdout_err[-50:])
+ asyncprocess.send_all(proc, fa)
+ hit = True
+ break
+ if not hit:
+ for question, answers in new_std_qa.items():
+ res = question.search(stdout_err)
+ if out and res:
+ fa = answers[0] % res.groupdict()
+ # cycle through list of answers
+ last_answer = answers.pop(0)
+ answers.append(last_answer)
+ _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers)
+
+ _log.debug("run_cmd_qa answer %s std question %s out %s",
+ fa, question.pattern, stdout_err[-50:])
+ asyncprocess.send_all(proc, fa)
+ hit = True
+ break
+ if not hit:
+ if len(stdout_err) > old_len_out:
+ old_len_out = len(stdout_err)
+ else:
+ noqa = False
+ for r in new_no_qa:
+ if r.search(stdout_err):
+ _log.debug("runqanda: noQandA found for out %s", stdout_err[-50:])
+ noqa = True
+ if not noqa:
+ hit_count += 1
+ else:
+ hit_count = 0
+ else:
+ hit_count = 0
+
+ if hit_count > maxhits:
+ # explicitly kill the child process before exiting
+ try:
+ os.killpg(proc.pid, signal.SIGKILL)
+ os.kill(proc.pid, signal.SIGKILL)
+ except OSError as err:
+ _log.debug("run_cmd_qa exception caught when killing child process: %s", err)
+ _log.debug("run_cmd_qa: full stdouterr: %s", stdout_err)
+ raise EasyBuildError("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s",
+ cmd, maxhits, stdout_err[-500:])
+
+ # the sleep below is required to avoid exiting on unknown 'questions' too early (see above)
+ time.sleep(1)
+ ec = proc.poll()
+
+ # Process stopped. Read all remaining data
+ try:
+ if proc.stdout:
+ out = get_output_from_process(proc, print_deprecation_warning=False)
+ stdout_err += out
+ if cmd_log:
+ cmd_log.write(out)
+ except IOError as err:
+ _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err)
+
+ run_hook_kwargs.update({
+ 'interactive': True,
+ 'exit_code': ec,
+ 'output': stdout_err,
+ })
+ run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
+
+ if trace:
+ trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
+
+ try:
+ os.chdir(cwd)
+ except OSError as err:
+ raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err)
+
+ return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok, regexp, print_deprecation_warning=False)
+
+
+def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp, print_deprecation_warning=True):
+ """
+ Parse command output and construct return value.
+ :param cmd: executed command
+ :param stdouterr: combined stdout/stderr of executed command
+ :param ec: exit code of executed command
+ :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
+ :param log_all: always log command output and exit code
+ :param log_ok: only run output/exit code for failing commands (exit code non-zero)
+ :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
+ """
+
+ if print_deprecation_warning:
+ _log.deprecated("parse_cmd_output is deprecated, you should stop using it", '6.0')
+
+ if strictness == IGNORE:
+ check_ec = False
+ fail_on_error_match = False
+ elif strictness == WARN:
+ check_ec = True
+ fail_on_error_match = False
+ elif strictness == ERROR:
+ check_ec = True
+ fail_on_error_match = True
+ else:
+ raise EasyBuildError("invalid strictness setting: %s", strictness)
+
+ # allow for overriding the regexp setting
+ if not regexp:
+ fail_on_error_match = False
+
+ if ec and (log_all or log_ok):
+ # We don't want to error if the user doesn't care
+ if check_ec:
+ raise EasyBuildError('cmd "%s" exited with exit code %s and output:\n%s', cmd, ec, stdouterr)
+ else:
+ _log.warning('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
+ elif not ec:
+ if log_all:
+ _log.info('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
+ else:
+ _log.debug('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
+
+ # parse the stdout/stderr for errors when strictness dictates this or when regexp is passed in
+ if fail_on_error_match or regexp:
+ res = parse_log_for_error(stdouterr, regexp, stdout=False, print_deprecation_warning=False)
+ if res:
+ errors = "\n\t" + "\n\t".join([r[0] for r in res])
+ error_str = "error" if len(res) == 1 else "errors"
+ if fail_on_error_match:
+ raise EasyBuildError("Found %s %s in output of %s:%s", len(res), error_str, cmd, errors)
+ else:
+ _log.warning("Found %s potential %s (some may be harmless) in output of %s:%s",
+ len(res), error_str, cmd, errors)
+
+ if simple:
+ if ec:
+ # If the user does not care -> will return true
+ return not check_ec
+ else:
+ return True
+ else:
+ # Because we are not running in simple mode, we return the output and ec to the user
+ return (stdouterr, ec)
+
+
+def parse_log_for_error(txt, regExp=None, stdout=True, msg=None, print_deprecation_warning=True):
+ """
+ txt is multiline string.
+ - in memory
+ regExp is a one-line regular expression
+ - default
+ """
+
+ if print_deprecation_warning:
+ _log.deprecated("parse_log_for_error is deprecated, you should stop using it", '6.0')
+
+ global errors_found_in_log
+
+ if regExp and isinstance(regExp, bool):
+ regExp = r"(?>> _env_to_boolean('NO_FOOBAR')
False
"""
- if varname not in os.environ:
+ try:
+ return os.environ[varname].lower() in ('1', 'yes', 'true', 'y')
+ except KeyError:
return default
- else:
- return os.environ.get(varname).lower() in ('1', 'yes', 'true', 'y')
OPTIMIZED_ANSWER = "not available in optimized mode"
@@ -345,7 +345,7 @@ def deprecated(self, msg, cur_ver, max_ver, depth=2, exception=None, log_callbac
if loose_cv.version[:depth] >= loose_mv.version[:depth]:
self.raiseException("DEPRECATED (since v%s) functionality used: %s" % (max_ver, msg), exception=exception)
else:
- deprecation_msg = "Deprecated functionality, will no longer work in v%s: %s" % (max_ver, msg)
+ deprecation_msg = "Deprecated functionality, will no longer work in EasyBuild v%s: %s" % (max_ver, msg)
log_callback(deprecation_msg)
def _handleFunction(self, function, levelno, **kwargs):
@@ -910,16 +910,15 @@ def resetroot():
_default_logTo = None
if 'FANCYLOG_SERVER' in os.environ:
server = os.environ['FANCYLOG_SERVER']
- port = DEFAULT_UDP_PORT
if ':' in server:
server, port = server.split(':')
+ else:
+ port = DEFAULT_UDP_PORT
# maybe the port was specified in the FANCYLOG_SERVER_PORT env var. this takes precedence
- if 'FANCYLOG_SERVER_PORT' in os.environ:
- port = int(os.environ['FANCYLOG_SERVER_PORT'])
- port = int(port)
+ port = os.environ.get('FANCYLOG_SERVER_PORT', port)
- logToUDP(server, port)
+ logToUDP(server, int(port))
_default_logTo = logToUDP
else:
# log to screen by default
diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py
index 4b99eab61d..c01035e391 100644
--- a/easybuild/base/generaloption.py
+++ b/easybuild/base/generaloption.py
@@ -1,5 +1,5 @@
#
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -100,11 +100,16 @@ def what_str_list_tuple(name):
"""Given name, return separator, class and helptext wrt separator.
(Currently supports strlist, strtuple, pathlist, pathtuple)
"""
- sep = ','
- helpsep = 'comma'
if name.startswith('path'):
sep = os.pathsep
helpsep = 'pathsep'
+ elif name.startswith('url'):
+ # | is one of the only characters not in the grammar for URIs (RFC3986)
+ sep = '|'
+ helpsep = '|'
+ else:
+ sep = ','
+ helpsep = 'comma'
klass = None
if name.endswith('list'):
@@ -185,6 +190,7 @@ class ExtOption(CompleterOption):
- strlist, strtuple : convert comma-separated string in a list resp. tuple of strings
- pathlist, pathtuple : using os.pathsep, convert pathsep-separated string in a list resp. tuple of strings
- the path separator is OS-dependent
+ - urllist, urltuple: convert string seperated by '|' to a list resp. tuple of strings
"""
EXTEND_SEPARATOR = ','
@@ -201,8 +207,9 @@ class ExtOption(CompleterOption):
TYPED_ACTIONS = Option.TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR
ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS
- TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']]
- TYPE_CHECKER = dict([(x, check_str_list_tuple) for x in TYPE_STRLIST] + list(Option.TYPE_CHECKER.items()))
+ TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path', 'url']]
+ TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST}
+ TYPE_CHECKER.update(Option.TYPE_CHECKER)
TYPES = tuple(TYPE_STRLIST + list(Option.TYPES))
BOOLEAN_ACTIONS = ('store_true', 'store_false',) + EXTOPTION_LOG
@@ -810,7 +817,7 @@ def get_env_options(self):
epilogprefixtxt += "eg. --some-opt is same as setting %(prefix)s_SOME_OPT in the environment."
self.epilog.append(epilogprefixtxt % {'prefix': self.envvar_prefix})
- candidates = dict([(k, v) for k, v in os.environ.items() if k.startswith("%s_" % self.envvar_prefix)])
+ candidates = {k: v for k, v in os.environ.items() if k.startswith("%s_" % self.envvar_prefix)}
for opt in self._get_all_options():
if opt._long_opts is None:
diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py
index 7a46c49921..ba0f075cf2 100644
--- a/easybuild/base/optcomplete.py
+++ b/easybuild/base/optcomplete.py
@@ -107,6 +107,7 @@
from optparse import OptionParser, Option
from pprint import pformat
+from easybuild.tools.filetools import get_cwd
from easybuild.tools.utilities import shell_quote
debugfn = None # for debugging only
@@ -537,7 +538,7 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
# Note: this will get filtered properly below.
completer_kwargs = {
- 'pwd': os.getcwd(),
+ 'pwd': get_cwd(),
'cline': cline,
'cpoint': cpoint,
'prefix': prefix,
diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py
index 9d3cea9d3c..24a2ec4cbf 100644
--- a/easybuild/base/testing.py
+++ b/easybuild/base/testing.py
@@ -1,5 +1,5 @@
#
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/__init__.py b/easybuild/framework/__init__.py
index 019e507d79..f03298abca 100644
--- a/easybuild/framework/__init__.py
+++ b/easybuild/framework/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 4482b0a71a..5fcd22e3d2 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -68,21 +68,22 @@
from easybuild.framework.easyconfig.tools import dump_env_easyblock, get_paths_for
from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
from easybuild.framework.extension import Extension, resolve_exts_filter_template
-from easybuild.tools import LooseVersion, config, run
+from easybuild.tools import LooseVersion, config
from easybuild.tools.build_details import get_build_stats
-from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, dry_run_msg, dry_run_warning, dry_run_set_dirs
from easybuild.tools.build_log import print_error, print_msg, print_warning
-from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
+from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES, PYTHONPATH, EBPYTHONPREFIXES
from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
+from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa
from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
from easybuild.tools.config import install_path, log_path, package_path, source_paths
from easybuild.tools.environment import restore_env, sanitize_env
-from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256
+from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256
from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info
from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file
-from easybuild.tools.filetools import encode_class_name, extract_file
-from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
+from easybuild.tools.filetools import encode_class_name, extract_file, find_backup_name_candidate
+from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url
from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
@@ -106,9 +107,6 @@
from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
-
-EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io'
-
DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64')
MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP]
@@ -151,7 +149,7 @@ def __init__(self, ec):
"""
# keep track of original working directory, so we can go back there
- self.orig_workdir = os.getcwd()
+ self.orig_workdir = get_cwd()
# dict of all hooks (mapping of name to function)
self.hooks = load_hooks(build_option('hooks'))
@@ -666,20 +664,24 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
src_path = ext_src['src']
src_fn = os.path.basename(src_path)
- # report both MD5 and SHA256 checksums, since both are valid default checksum types
- for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
+ src_checksums = {}
+ for checksum_type in [CHECKSUM_TYPE_SHA256]:
src_checksum = compute_checksum(src_path, checksum_type=checksum_type)
+ src_checksums[checksum_type] = src_checksum
self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum)
# verify checksum (if provided)
self.log.debug('Verifying checksums for extension source...')
fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0)
- if verify_checksum(src_path, fn_checksum):
+ if verify_checksum(src_path, fn_checksum, src_checksums):
self.log.info('Checksum for extension source %s verified', src_fn)
elif build_option('ignore_checksums'):
print_warning("Ignoring failing checksum verification for %s" % src_fn)
else:
- raise EasyBuildError('Checksum verification for extension source %s failed', src_fn)
+ raise EasyBuildError(
+ 'Checksum verification for extension source %s failed', src_fn,
+ exit_code=EasyBuildExit.FAIL_CHECKSUM
+ )
# locate extension patches (if any), and verify checksums
ext_patches = ext_options.get('patches', [])
@@ -693,12 +695,13 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
ext_src.update({'patches': ext_patches})
if verify_checksums:
+ computed_checksums = {}
for patch in ext_patches:
patch = patch['path']
- # report both MD5 and SHA256 checksums,
- # since both are valid default checksum types
- for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
+ computed_checksums[patch] = {}
+ for checksum_type in [CHECKSUM_TYPE_SHA256]:
checksum = compute_checksum(patch, checksum_type=checksum_type)
+ computed_checksums[patch][checksum_type] = checksum
self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
# verify checksum (if provided)
@@ -708,13 +711,15 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
patch_fn = os.path.basename(patch)
checksum = self.get_checksum_for(checksums, filename=patch_fn, index=idx+1)
- if verify_checksum(patch, checksum):
+ if verify_checksum(patch, checksum, computed_checksums[patch]):
self.log.info('Checksum for extension patch %s verified', patch_fn)
elif build_option('ignore_checksums'):
print_warning("Ignoring failing checksum verification for %s" % patch_fn)
else:
- raise EasyBuildError("Checksum verification for extension patch %s failed",
- patch_fn)
+ raise EasyBuildError(
+ "Checksum verification for extension patch %s failed", patch_fn,
+ exit_code=EasyBuildExit.FAIL_CHECKSUM
+ )
else:
self.log.debug('No patches found for extension %s.' % ext_name)
@@ -747,7 +752,9 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
"""
srcpaths = source_paths()
- update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
+ # We don't account for the checksums file in the progress bar
+ if filename != 'checksum.json':
+ update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
if alt_location is None:
location = self.name
@@ -783,12 +790,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
return fullpath
except IOError as err:
+ msg = f"Downloading file {filename} from url {url} to {fullpath} failed: {err}"
if not warning_only:
- raise EasyBuildError("Downloading file %s "
- "from url %s to %s failed: %s", filename, url, fullpath, err)
+ raise EasyBuildError(msg, exit_code=EasyBuildExit.FAIL_DOWNLOAD)
else:
- self.log.warning("Downloading file %s "
- "from url %s to %s failed: %s", filename, url, fullpath, err)
+ self.log.warning(msg)
return None
else:
@@ -861,115 +867,124 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
if self.dry_run:
self.dry_run_msg(" * %s found at %s", filename, foundfile)
return foundfile
- elif no_download:
+
+ if no_download:
if self.dry_run:
self.dry_run_msg(" * %s (MISSING)", filename)
return filename
- else:
- if not warning_only:
- raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
- "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
- else:
- self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... "
- "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
- return None
- elif git_config:
- return get_source_tarball_from_git(filename, targetdir, git_config)
- else:
- # try and download source files from specified source URLs
- if urls:
- source_urls = urls[:]
- else:
- source_urls = []
- source_urls.extend(self.cfg['source_urls'])
- # add https://sources.easybuild.io as fallback source URL
- source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location))
+ failedpaths_msg = "\n * ".join([""]+failedpaths)
+ file_notfound_msg = (
+ f"Couldn't find file '{filename}' anywhere, and downloading it is disabled... "
+ f"Paths attempted (in order): {failedpaths_msg}"
+ )
- mkdir(targetdir, parents=True)
+ if warning_only:
+ self.log.warning(file_notfound_msg)
+ return None
- for url in source_urls:
+ raise EasyBuildError(file_notfound_msg, exit_code=EasyBuildExit.MISSING_SOURCES)
- if extension:
- targetpath = os.path.join(targetdir, "extensions", filename)
- else:
- targetpath = os.path.join(targetdir, filename)
+ if git_config:
+ return get_source_tarball_from_git(filename, targetdir, git_config)
- url_filename = download_filename or filename
+ # try and download source files from specified source URLs
+ if urls:
+ source_urls = urls[:]
+ else:
+ source_urls = []
+ source_urls.extend(self.cfg['source_urls'])
- if isinstance(url, str):
- if url[-1] in ['=', '/']:
- fullurl = "%s%s" % (url, url_filename)
- else:
- fullurl = "%s/%s" % (url, url_filename)
- elif isinstance(url, tuple):
- # URLs that require a suffix, e.g., SourceForge download links
- # e.g. http://sourceforge.net/projects/math-atlas/files/Stable/3.8.4/atlas3.8.4.tar.bz2/download
- fullurl = "%s/%s/%s" % (url[0], url_filename, url[1])
- else:
- self.log.warning("Source URL %s is of unknown type, so ignoring it." % url)
- continue
+ # Add additional URLs as configured.
+ for url in build_option("extra_source_urls"):
+ url += "/" + name_letter + "/" + location
+ source_urls.append(url)
- # PyPI URLs may need to be converted due to change in format of these URLs,
- # cfr. https://bitbucket.org/pypa/pypi/issues/438
- if PYPI_PKG_URL_PATTERN in fullurl and not is_alt_pypi_url(fullurl):
- alt_url = derive_alt_pypi_url(fullurl)
- if alt_url:
- _log.debug("Using alternate PyPI URL for %s: %s", fullurl, alt_url)
- fullurl = alt_url
- else:
- _log.debug("Failed to derive alternate PyPI URL for %s, so retaining the original", fullurl)
+ mkdir(targetdir, parents=True)
- if self.dry_run:
- self.dry_run_msg(" * %s will be downloaded to %s", filename, targetpath)
- if extension and urls:
- # extensions typically have custom source URLs specified, only mention first
- self.dry_run_msg(" (from %s, ...)", fullurl)
- downloaded = True
+ for url in source_urls:
- else:
- self.log.debug("Trying to download file %s from %s to %s ..." % (filename, fullurl, targetpath))
- downloaded = False
- try:
- if download_file(filename, fullurl, targetpath):
- downloaded = True
+ if extension:
+ targetpath = os.path.join(targetdir, "extensions", filename)
+ else:
+ targetpath = os.path.join(targetdir, filename)
- except IOError as err:
- self.log.debug("Failed to download %s from %s: %s" % (filename, url, err))
- failedpaths.append(fullurl)
- continue
+ url_filename = download_filename or filename
- if downloaded:
- # if fetching from source URL worked, we're done
- self.log.info("Successfully downloaded source file %s from %s" % (filename, fullurl))
- return targetpath
+ if isinstance(url, str):
+ if url[-1] in ['=', '/']:
+ fullurl = "%s%s" % (url, url_filename)
else:
- failedpaths.append(fullurl)
+ fullurl = "%s/%s" % (url, url_filename)
+ elif isinstance(url, tuple):
+ # URLs that require a suffix, e.g., SourceForge download links
+ # e.g. http://sourceforge.net/projects/math-atlas/files/Stable/3.8.4/atlas3.8.4.tar.bz2/download
+ fullurl = "%s/%s/%s" % (url[0], url_filename, url[1])
+ else:
+ self.log.warning("Source URL %s is of unknown type, so ignoring it." % url)
+ continue
+
+ # PyPI URLs may need to be converted due to change in format of these URLs,
+ # cfr. https://bitbucket.org/pypa/pypi/issues/438
+ if PYPI_PKG_URL_PATTERN in fullurl and not is_alt_pypi_url(fullurl):
+ alt_url = derive_alt_pypi_url(fullurl)
+ if alt_url:
+ _log.debug("Using alternative PyPI URL for %s: %s", fullurl, alt_url)
+ fullurl = alt_url
+ else:
+ _log.debug("Failed to derive alternative PyPI URL for %s, so retaining the original",
+ fullurl)
if self.dry_run:
- self.dry_run_msg(" * %s (MISSING)", filename)
- return filename
+ self.dry_run_msg(" * %s will be downloaded to %s", filename, targetpath)
+ if extension and urls:
+ # extensions typically have custom source URLs specified, only mention first
+ self.dry_run_msg(" (from %s, ...)", fullurl)
+ downloaded = True
+
else:
- error_msg = "Couldn't find file %s anywhere, "
- if download_instructions is None:
- download_instructions = self.cfg['download_instructions']
- if download_instructions is not None and download_instructions != "":
- msg = "\nDownload instructions:\n\n" + indent(download_instructions, ' ') + '\n\n'
- msg += "Make the files available in the active source path: %s\n" % ':'.join(source_paths())
- print_msg(msg, prefix=False, stderr=True)
- error_msg += "please follow the download instructions above, and make the file available "
- error_msg += "in the active source path (%s)" % ':'.join(source_paths())
- else:
- # flatten list to string with '%' characters escaped (literal '%' desired in 'sprintf')
- failedpaths_msg = ', '.join(failedpaths).replace('%', '%%')
- error_msg += "and downloading it didn't work either... "
- error_msg += "Paths attempted (in order): %s " % failedpaths_msg
+ self.log.debug("Trying to download file %s from %s to %s ..." % (filename, fullurl, targetpath))
+ downloaded = False
+ try:
+ if download_file(filename, fullurl, targetpath):
+ downloaded = True
- if not warning_only:
- raise EasyBuildError(error_msg, filename)
- else:
- self.log.warning(error_msg, filename)
- return None
+ except IOError as err:
+ self.log.debug("Failed to download %s from %s: %s" % (filename, url, err))
+ failedpaths.append(fullurl)
+ continue
+
+ if downloaded:
+ # if fetching from source URL worked, we're done
+ self.log.info("Successfully downloaded source file %s from %s" % (filename, fullurl))
+ return targetpath
+ else:
+ failedpaths.append(fullurl)
+
+ if self.dry_run:
+ self.dry_run_msg(" * %s (MISSING)", filename)
+ return filename
+ else:
+ error_msg = "Couldn't find file %s anywhere, "
+ if download_instructions is None:
+ download_instructions = self.cfg['download_instructions']
+ if download_instructions is not None and download_instructions != "":
+ msg = "\nDownload instructions:\n\n" + indent(download_instructions, ' ') + '\n\n'
+ msg += "Make the files available in the active source path: %s\n" % ':'.join(source_paths())
+ print_msg(msg, prefix=False, stderr=True)
+ error_msg += "please follow the download instructions above, and make the file available "
+ error_msg += "in the active source path (%s)" % ':'.join(source_paths())
+ else:
+ # flatten list to string with '%' characters escaped (literal '%' desired in 'sprintf')
+ failedpaths_msg = ', '.join(failedpaths).replace('%', '%%')
+ error_msg += "and downloading it didn't work either... "
+ error_msg += "Paths attempted (in order): %s " % failedpaths_msg
+
+ if not warning_only:
+ raise EasyBuildError(error_msg, filename, exit_code=EasyBuildExit.FAIL_DOWNLOAD)
+ else:
+ self.log.warning(error_msg, filename)
+ return None
#
# GETTER/SETTER UTILITY FUNCTIONS
@@ -1062,7 +1077,7 @@ def make_builddir(self):
self.log.info("Overriding 'cleanupoldinstall' (to False), 'cleanupoldbuild' (to True) "
"and 'keeppreviousinstall' because we're building in the installation directory.")
# force cleanup before installation
- if build_option('module_only'):
+ if build_option('module_only') or self.cfg['module_only']:
self.log.debug("Disabling cleanupoldbuild because we run as module-only")
self.cfg['cleanupoldbuild'] = False
else:
@@ -1133,7 +1148,7 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False):
if self.cfg['keeppreviousinstall']:
self.log.info("Keeping old directory %s (hopefully you know what you are doing)", dir_name)
return
- elif build_option('module_only'):
+ elif build_option('module_only') or self.cfg['module_only']:
self.log.info("Not touching existing directory %s in module-only mode...", dir_name)
elif clean:
remove_dir(dir_name)
@@ -1204,6 +1219,8 @@ def make_devel_module(self, create_in_builddir=False):
# these should be all the dependencies and we should load them
recursive_unload = self.cfg['recursive_module_unload']
depends_on = self.cfg['module_depends_on']
+ if depends_on is not None:
+ self.log.deprecated("'module_depends_on' easyconfig parameter should not be used anymore", '6.0')
for key in os.environ:
# legacy support
if key.startswith(DEVEL_ENV_VAR_NAME_PREFIX):
@@ -1331,6 +1348,8 @@ def make_module_dep(self, unload_info=None):
# include load statements for retained dependencies
recursive_unload = self.cfg['recursive_module_unload']
depends_on = self.cfg['module_depends_on']
+ if depends_on is not None:
+ self.log.deprecated("'module_depends_on' easyconfig parameter should not be used anymore", '6.0')
loads = []
for dep in deps:
unload_modules = []
@@ -1359,7 +1378,7 @@ def make_module_dep(self, unload_info=None):
multi_dep_mod_names[dep['name']].append(dep['short_mod_name'])
multi_dep_load_defaults = []
- for depname, depmods in sorted(multi_dep_mod_names.items()):
+ for _, depmods in sorted(multi_dep_mod_names.items()):
stmt = self.module_generator.load_module(depmods[0], multi_dep_mods=depmods,
recursive_unload=recursive_unload,
depends_on=depends_on)
@@ -1375,6 +1394,46 @@ def make_module_description(self):
"""
return self.module_generator.get_description()
+ def make_module_pythonpath(self):
+ """
+ Add lines for module file to update $PYTHONPATH or $EBPYTHONPREFIXES,
+ if they aren't already present and the standard lib/python*/site-packages subdirectory exists
+ """
+ if os.path.isfile(os.path.join(self.installdir, 'bin', 'python')): # only needed when not a python install
+ return []
+
+ python_subdir_pattern = os.path.join(self.installdir, 'lib', 'python*', 'site-packages')
+ candidate_paths = (os.path.relpath(path, self.installdir) for path in glob.glob(python_subdir_pattern))
+ python_paths = [path for path in candidate_paths if re.match(r'lib/python\d+\.\d+/site-packages', path)]
+ if not python_paths:
+ return []
+
+ # determine whether Python is a runtime dependency;
+ # if so, we assume it was installed with EasyBuild, and hence is aware of $EBPYTHONPREFIXES
+ runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
+
+ # don't use $EBPYTHONPREFIXES unless we can and it's preferred or necesary (due to use of multi_deps)
+ use_ebpythonprefixes = False
+ multi_deps = self.cfg['multi_deps']
+
+ if 'Python' in runtime_deps:
+ self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...")
+
+ if build_option('prefer_python_search_path') == EBPYTHONPREFIXES:
+ self.log.info("Preferred Python search path is $EBPYTHONPREFIXES, so using that")
+ use_ebpythonprefixes = True
+
+ elif multi_deps and 'Python' in multi_deps:
+ self.log.info("Python is listed in 'multi_deps', so using $EBPYTHONPREFIXES instead of $PYTHONPATH")
+ use_ebpythonprefixes = True
+
+ if use_ebpythonprefixes:
+ path = '' # EBPYTHONPREFIXES are relative to the install dir
+ lines = self.module_generator.prepend_paths(EBPYTHONPREFIXES, path, warn_exists=False)
+ else:
+ lines = self.module_generator.prepend_paths(PYTHONPATH, python_paths, warn_exists=False)
+ return [lines] if lines else []
+
def make_module_extra(self, altroot=None, altversion=None):
"""
Set extra stuff in module file, e.g. $EBROOT*, $EBVERSION*, etc.
@@ -1407,21 +1466,29 @@ def make_module_extra(self, altroot=None, altversion=None):
for (key, value) in self.cfg['modextravars'].items():
lines.append(self.module_generator.set_environment(key, value))
- for (key, value) in self.cfg['modextrapaths'].items():
- if isinstance(value, str):
- value = [value]
- elif not isinstance(value, (tuple, list)):
- raise EasyBuildError("modextrapaths dict value %s (type: %s) is not a list or tuple",
- value, type(value))
- lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path']))
-
- for (key, value) in self.cfg['modextrapaths_append'].items():
- if isinstance(value, str):
- value = [value]
- elif not isinstance(value, (tuple, list)):
- raise EasyBuildError("modextrapaths_append dict value %s (type: %s) is not a list or tuple",
- value, type(value))
- lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path']))
+ for extrapaths_type, prepend in [('modextrapaths', True), ('modextrapaths_append', False)]:
+ allow_abs = self.cfg['allow_prepend_abs_path'] if prepend else self.cfg['allow_append_abs_path']
+
+ for (key, value) in self.cfg[extrapaths_type].items():
+ if not isinstance(value, (tuple, list, dict, str)):
+ raise EasyBuildError(
+ f"{extrapaths_type} dict value '{value}' (type {type(value)}) is not a 'list, dict or str'"
+ )
+
+ try:
+ paths = value['paths']
+ delim = value['delimiter']
+ except KeyError:
+ raise EasyBuildError(f'{extrapaths_type} dict "{value}" lacks "paths" or "delimiter" items')
+ except TypeError:
+ paths = value
+ delim = ':'
+
+ lines.append(
+ self.module_generator.update_paths(key, paths, prepend=prepend, delim=delim, allow_abs=allow_abs)
+ )
+ # add lines to update $PYTHONPATH or $EBPYTHONPREFIXES
+ lines.extend(self.make_module_pythonpath())
modloadmsg = self.cfg['modloadmsg']
if modloadmsg:
@@ -1748,7 +1815,8 @@ def _make_extension_list(self):
if isinstance(ext, str):
exts_list.append((resolve_template(ext, self.cfg.template_values), ))
else:
- exts_list.append((resolve_template(ext[0], self.cfg.template_values), ext[1]))
+ exts_list.append((resolve_template(ext[0], self.cfg.template_values),
+ resolve_template(ext[1], self.cfg.template_values)))
return exts_list
def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True):
@@ -1763,10 +1831,7 @@ def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True):
return ext_sep.join(exts_list)
def prepare_for_extensions(self):
- """
- Also do this before (eg to set the template)
- """
- pass
+ """Ran before installing extensions (eg to set templates)"""
def skip_extensions(self):
"""
@@ -1802,7 +1867,7 @@ def skip_extensions_sequential(self, exts_filter):
cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst)
res = run_shell_cmd(cmd, stdin=stdin, fail_on_error=False, hidden=True)
self.log.info(f"exts_filter result for {ext_inst.name}: exit code {res.exit_code}; output: {res.output}")
- if res.exit_code == 0:
+ if res.exit_code == EasyBuildExit.SUCCESS:
print_msg(f"skipping extension {ext_inst.name}", silent=self.silent, log=self.log)
else:
self.log.info(f"Not skipping {ext_inst.name}")
@@ -1818,7 +1883,6 @@ def skip_extensions_parallel(self, exts_filter):
Skip already installed extensions (checking in parallel),
by removing them from list of Extension instances to install (self.ext_instances).
"""
- self.log.experimental("Skipping installed extensions in parallel")
print_msg("skipping installed extensions (in parallel)", log=self.log)
installed_exts_ids = []
@@ -1838,7 +1902,7 @@ def skip_extensions_parallel(self, exts_filter):
idx = res.task_id
ext_name = self.ext_instances[idx].name
self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}")
- if res.exit_code == 0:
+ if res.exit_code == EasyBuildExit.SUCCESS:
print_msg(f"skipping extension {ext_name}", log=self.log)
installed_exts_ids.append(idx)
@@ -1857,7 +1921,15 @@ def skip_extensions_parallel(self, exts_filter):
self.ext_instances = retained_ext_instances
- def install_extensions(self, install=True):
+ def install_extensions(self, *args, **kwargs):
+ """[DEPRECATED] Install extensions."""
+ self.log.deprecated(
+ "EasyBlock.install_extensions() is deprecated, use EasyBlock.install_all_extensions() instead.",
+ '6.0',
+ )
+ self.install_all_extensions(*args, **kwargs)
+
+ def install_all_extensions(self, install=True):
"""
Install extensions.
@@ -1867,13 +1939,12 @@ def install_extensions(self, install=True):
self.log.debug("List of loaded modules: %s", self.modules_tool.list())
if build_option('parallel_extensions_install'):
- self.log.experimental("installing extensions in parallel")
try:
self.install_extensions_parallel(install=install)
except NotImplementedError:
# If parallel extension install is not supported for this type of extension then install sequentially
msg = "Parallel extensions install not supported for %s - using sequential install" % self.name
- self.log.experimental(msg)
+ self.log.info(msg)
self.install_extensions_sequential(install=install)
else:
self.install_extensions_sequential(install=install)
@@ -1889,7 +1960,6 @@ def install_extensions_sequential(self, install=True):
exts_cnt = len(self.ext_instances)
for idx, ext in enumerate(self.ext_instances):
-
self.log.info("Starting extension %s", ext.name)
run_hook(SINGLE_EXTENSION, self.hooks, pre_step_hook=True, args=[ext])
@@ -1921,15 +1991,15 @@ def install_extensions_sequential(self, install=True):
ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
rpath_filter_dirs=self.rpath_filter_dirs)
- # real work
+ # actual installation of the extension
if install:
try:
- ext.prerun()
+ ext.install_extension_substep("pre_install_extension")
with self.module_generator.start_module_creation():
- txt = ext.run()
+ txt = ext.install_extension_substep("install_extension")
if txt:
self.module_extra_extensions += txt
- ext.postrun()
+ ext.install_extension_substep("post_install_extension")
finally:
if not self.dry_run:
ext_duration = datetime.now() - start_time
@@ -1992,11 +2062,11 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
for ext in running_exts[:]:
if self.dry_run or ext.async_cmd_task.done():
res = ext.async_cmd_task.result()
- if res.exit_code == 0:
+ if res.exit_code == EasyBuildExit.SUCCESS:
self.log.info(f"Installation of extension {ext.name} completed!")
# run post-install method for extension from same working dir as installation of extension
cwd = change_dir(res.work_dir)
- ext.postrun()
+ ext.install_extension_substep("post_install_extension")
change_dir(cwd)
running_exts.remove(ext)
installed_ext_names.append(ext.name)
@@ -2048,20 +2118,33 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
# if some of the required dependencies are not installed yet, requeue this extension
elif pending_deps:
- # make sure all required dependencies are actually going to be installed,
- # to avoid getting stuck in an infinite loop!
+ # check whether all required dependency extensions are actually going to be installed;
+ # if not, we assume that they are provided by dependencies;
missing_deps = [x for x in required_deps if x not in all_ext_names]
if missing_deps:
- raise EasyBuildError("Missing required dependencies for %s are not going to be installed: %s",
- ext.name, ', '.join(missing_deps))
- else:
- self.log.info("Required dependencies missing for extension %s (%s), adding it back to queue...",
- ext.name, ', '.join(pending_deps))
+ msg = f"Missing required extensions for {ext.name} not found "
+ msg += "in list of extensions being installed, let's assume they are provided by "
+ msg += "dependencies and proceed: " + ', '.join(missing_deps)
+ self.log.info(msg)
+
+ msg = f"Pending dependencies for {ext.name} before taking into account missing dependencies: "
+ self.log.debug(msg + ', '.join(pending_deps))
+ pending_deps = [x for x in pending_deps if x not in missing_deps]
+ msg = f"Pending dependencies for {ext.name} after taking into account missing dependencies: "
+ self.log.debug(msg + ', '.join(pending_deps))
+
+ if pending_deps:
+ msg = f"Required dependencies not installed yet for extension {ext.name} ("
+ msg += ', '.join(pending_deps)
+ msg += "), adding it back to queue..."
+ self.log.info(msg)
# purposely adding extension back in the queue at Nth place rather than at the end,
# since we assume that the required dependencies will be installed soon...
exts_queue.insert(max_iter, ext)
- else:
+ # list of pending dependencies may be empty now after taking into account required extensions
+ # that are not being installed above, so extension may be ready to install
+ if not pending_deps:
tup = (ext.name, ext.version or '')
print_msg("starting installation of extension %s %s..." % tup, silent=self.silent, log=self.log)
@@ -2070,8 +2153,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
rpath_filter_dirs=self.rpath_filter_dirs)
if install:
- ext.prerun()
- ext.async_cmd_task = ext.run_async(thread_pool)
+ ext.install_extension_substep("pre_install_extension")
+ ext.async_cmd_task = ext.install_extension_substep("install_extension_async", thread_pool)
running_exts.append(ext)
self.log.info(f"Started installation of extension {ext.name} in the background...")
update_exts_progress_bar_helper(running_exts, 0)
@@ -2108,7 +2191,7 @@ def guess_start_dir(self):
start_dir = ''
# do not use the specified 'start_dir' when running as --module-only as
# the directory will not exist (extract_step is skipped)
- if self.start_dir and not build_option('module_only'):
+ if self.start_dir and not build_option('module_only') and not self.cfg['module_only']:
start_dir = self.start_dir
if not os.path.isabs(start_dir):
@@ -2192,9 +2275,9 @@ def handle_iterate_opts(self):
self.log.info("Current iteration index: %s", self.iter_idx)
# pop first element from all iterative easyconfig parameters as next value to use
- for opt in self.iter_opts:
- if len(self.iter_opts[opt]) > self.iter_idx:
- self.cfg[opt] = self.iter_opts[opt][self.iter_idx]
+ for opt, value in self.iter_opts.items():
+ if len(value) > self.iter_idx:
+ self.cfg[opt] = value[self.iter_idx]
else:
self.cfg[opt] = '' # empty list => empty option as next value
self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt])))
@@ -2206,12 +2289,12 @@ def post_iter_step(self):
"""Restore options that were iterated over"""
# disable templating, since we're messing about with values in self.cfg
with self.cfg.disable_templating():
- for opt in self.iter_opts:
- self.cfg[opt] = self.iter_opts[opt]
+ for opt, value in self.iter_opts.items():
+ self.cfg[opt] = value
# also need to take into account extensions, since those were iterated over as well
for ext in self.ext_instances:
- ext.cfg[opt] = self.iter_opts[opt]
+ ext.cfg[opt] = value
self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt])
@@ -2343,7 +2426,7 @@ def check_readiness_step(self):
self.log.info("No module %s found. Not skipping anything." % self.full_mod_name)
# remove existing module file under --force (but only if --skip is not used)
- elif build_option('force') or build_option('rebuild'):
+ elif (build_option('force') or build_option('rebuild')) and not build_option('dump_env_script'):
self.remove_module_file()
def fetch_step(self, skip_checksums=False):
@@ -2402,8 +2485,7 @@ def fetch_step(self, skip_checksums=False):
# compute checksums for all source and patch files
if not (skip_checksums or self.dry_run):
for fil in self.src + self.patches:
- # report both MD5 and SHA256 checksums, since both are valid default checksum types
- for checksum_type in [CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256]:
+ for checksum_type in [CHECKSUM_TYPE_SHA256]:
fil[checksum_type] = compute_checksum(fil['path'], checksum_type=checksum_type)
self.log.info("%s checksum for %s: %s", checksum_type, fil['path'], fil[checksum_type])
@@ -2462,7 +2544,10 @@ def checksum_step(self):
elif build_option('ignore_checksums'):
print_warning("Ignoring failing checksum verification for %s" % fil['name'])
else:
- raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum'])
+ raise EasyBuildError(
+ "Checksum verification for %s using %s failed.", fil['path'], fil['checksum'],
+ exit_code=EasyBuildExit.FAIL_CHECKSUM
+ )
def check_checksums_for(self, ent, sub='', source_cnt=None):
"""
@@ -2472,7 +2557,7 @@ def check_checksums_for(self, ent, sub='', source_cnt=None):
checksum_issues = []
sources = ent.get('sources', [])
- patches = ent.get('patches', [])
+ patches = ent.get('patches', []) + ent.get('postinstallpatches', [])
checksums = ent.get('checksums', [])
# Single source should be re-wrapped as a list, and checksums with it
if isinstance(sources, dict):
@@ -2601,7 +2686,7 @@ def patch_step(self, beginpath=None, patches=None):
copy_patch = 'copy' in patch and 'sourcepath' not in patch
self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s",
- srcind, level, srcpathsuffix, copy)
+ srcind, level, srcpathsuffix, copy_patch)
if beginpath is None:
try:
@@ -2746,10 +2831,7 @@ def _test_step(self):
self.report_test_failure(error_msg)
def stage_install_step(self):
- """
- Install in a stage directory before actual installation.
- """
- pass
+ """Install in a stage directory before actual installation."""
def install_step(self):
"""Install built software (abstract method)."""
@@ -2787,7 +2869,6 @@ def init_ext_instances(self):
exts_cnt = len(self.exts)
self.update_exts_progress_bar("creating internal datastructures for extensions")
-
for idx, ext in enumerate(self.exts):
ext_name = ext['name']
self.log.debug("Creating class instance for extension %s...", ext_name)
@@ -2904,7 +2985,7 @@ def extensions_step(self, fetch=False, install=True):
if self.skip:
self.skip_extensions()
- self.install_extensions(install=install)
+ self.install_all_extensions(install=install)
# cleanup (unload fake module, remove fake module dir)
if fake_mod_data:
@@ -3119,8 +3200,13 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True):
fails = []
- # hard reset $LD_LIBRARY_PATH before running RPATH sanity check
- orig_env = env.unset_env_vars(['LD_LIBRARY_PATH'])
+ if build_option('strict_rpath_sanity_check'):
+ self.log.info("Unsetting $LD_LIBRARY_PATH since strict RPATH sanity check is enabled...")
+ # hard reset $LD_LIBRARY_PATH before running RPATH sanity check
+ orig_env = env.unset_env_vars(['LD_LIBRARY_PATH'])
+ else:
+ self.log.info("Not unsetting $LD_LIBRARY_PATH since strict RPATH sanity check is disabled...")
+ orig_env = None
ld_library_path = os.getenv('LD_LIBRARY_PATH', '(empty)')
self.log.debug(f"$LD_LIBRARY_PATH during RPATH sanity check: {ld_library_path}")
@@ -3128,6 +3214,7 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True):
self.log.debug(f"List of loaded modules: {modules_list}")
not_found_regex = re.compile(r'(\S+)\s*\=\>\s*not found')
+ lib_path_regex = re.compile(r'\S+\s*\=\>\s*(\S+)')
readelf_rpath_regex = re.compile('(RPATH)', re.M)
# List of libraries that should be exempt from the RPATH sanity check;
@@ -3173,14 +3260,23 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True):
fail_msg = f"Library {match} not found for {path}"
self.log.warning(fail_msg)
fails.append(fail_msg)
+
+ # if any libraries were not found, log whether dependency libraries have an RPATH section
+ if fails:
+ lib_paths = re.findall(lib_path_regex, out)
+ for lib_path in lib_paths:
+ self.log.info(f"Checking whether dependency library {lib_path} has RPATH section")
+ res = run_shell_cmd(f"readelf -d {lib_path}", fail_on_error=False)
+ if res.exit_code:
+ self.log.info(f"No RPATH section found in {lib_path}")
else:
self.log.debug(f"Output of 'ldd {path}' checked, looks OK")
# check whether RPATH section in 'readelf -d' output is there
if check_readelf_rpath:
fail_msg = None
- res = run_shell_cmd(f"readelf -d {path}", fail_on_error=False)
- if res.exit_code:
+ res = run_shell_cmd(f"readelf -d {path}", fail_on_error=False, hidden=True)
+ if res.exit_code != EasyBuildExit.SUCCESS:
fail_msg = f"Failed to run 'readelf -d {path}': {res.output}"
elif not readelf_rpath_regex.search(res.output):
fail_msg = f"No '(RPATH)' found in 'readelf -d' output for {path}: {out}"
@@ -3195,7 +3291,8 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True):
else:
self.log.debug(f"Not sanity checking files in non-existing directory {dirpath}")
- env.restore_env_vars(orig_env)
+ if orig_env:
+ env.restore_env_vars(orig_env)
return fails
@@ -3243,7 +3340,7 @@ def sanity_check_linked_shared_libs(self, subdirs=None):
required_libs.extend(self.cfg['required_linked_shared_libs'])
# early return if there are no banned/required libraries
- if not (banned_libs + required_libs):
+ if not banned_libs + required_libs:
self.log.info("No banned/required libraries specified")
return []
else:
@@ -3496,10 +3593,7 @@ def _sanity_check_step_extensions(self):
"""Sanity check on extensions (if any)."""
failed_exts = []
- if build_option('skip_extensions'):
- self.log.info("Skipping sanity check for extensions since skip-extensions is enabled...")
- return
- elif not self.ext_instances:
+ if not self.ext_instances:
# class instances for extensions may not be initialized yet here,
# for example when using --module-only or --sanity-check-only
self.prepare_for_extensions()
@@ -3618,6 +3712,7 @@ def xs2str(xs):
if not found:
sanity_check_fail_msg = "no %s found at %s in %s" % (typ, xs2str(xs), self.installdir)
self.sanity_check_fail_msgs.append(sanity_check_fail_msg)
+ self.exit_code = EasyBuildExit.FAIL_SANITY_CHECK
self.log.warning("Sanity check: %s", sanity_check_fail_msg)
trace_msg("%s %s found: %s" % (typ, xs2str(xs), ('FAILED', 'OK')[found]))
@@ -3640,19 +3735,23 @@ def xs2str(xs):
trace_msg(f"running command '{cmd}' ...")
res = run_shell_cmd(cmd, fail_on_error=False, hidden=True)
- if res.exit_code != 0:
- fail_msg = f"sanity check command {cmd} exited with code {res.exit_code} (output: {res.output})"
+ if res.exit_code != EasyBuildExit.SUCCESS:
+ fail_msg = f"sanity check command {cmd} failed with exit code {res.exit_code} (output: {res.output})"
self.sanity_check_fail_msgs.append(fail_msg)
+ self.exit_code = EasyBuildExit.FAIL_SANITY_CHECK
self.log.warning(f"Sanity check: {fail_msg}")
else:
self.log.info(f"sanity check command {cmd} ran successfully! (output: {res.output})")
- cmd_result_str = ('FAILED', 'OK')[res.exit_code == 0]
+ cmd_result_str = ('FAILED', 'OK')[res.exit_code == EasyBuildExit.SUCCESS]
trace_msg(f"result for command '{cmd}': {cmd_result_str}")
# also run sanity check for extensions (unless we are an extension ourselves)
if not extension:
- self._sanity_check_step_extensions()
+ if build_option('skip_extensions'):
+ self.log.info("Skipping sanity check for extensions since skip-extensions is enabled...")
+ else:
+ self._sanity_check_step_extensions()
linked_shared_lib_fails = self.sanity_check_linked_shared_libs()
if linked_shared_lib_fails:
@@ -3685,7 +3784,10 @@ def xs2str(xs):
# pass or fail
if self.sanity_check_fail_msgs:
- raise EasyBuildError("Sanity check failed: " + '\n'.join(self.sanity_check_fail_msgs))
+ raise EasyBuildError(
+ "Sanity check failed: " + '\n'.join(self.sanity_check_fail_msgs),
+ exit_code=EasyBuildExit.FAIL_SANITY_CHECK,
+ )
else:
self.log.debug("Sanity check passed!")
@@ -3717,7 +3819,7 @@ def cleanup_step(self):
# make sure we're out of the dir we're removing
change_dir(self.orig_workdir)
- self.log.info("Cleaning up builddir %s (in %s)", self.builddir, os.getcwd())
+ self.log.info("Cleaning up builddir %s (in %s)", self.builddir, get_cwd())
try:
remove_dir(self.builddir)
@@ -3787,8 +3889,13 @@ def make_module_step(self, fake=False):
for line in txt.split('\n'):
self.dry_run_msg(INDENT_4SPACES + line)
else:
- write_file(mod_filepath, txt)
- self.log.info("Module file %s written: %s", mod_filepath, txt)
+ try:
+ write_file(mod_filepath, txt)
+ self.log.info("Module file %s written: %s", mod_filepath, txt)
+ except EasyBuildError:
+ raise EasyBuildError(
+ f"Unable to write Module file {mod_filepath}", exit_code=EasyBuildExit.FAIL_MODULE_WRITE
+ )
# if backup module file is there, print diff with newly generated module file
if self.mod_file_backup and not fake:
@@ -3814,7 +3921,7 @@ def make_module_step(self, fake=False):
try:
self.make_devel_module()
except EasyBuildError as error:
- if build_option('module_only'):
+ if build_option('module_only') or self.cfg['module_only']:
self.log.info("Using --module-only so can recover from error: %s", error)
else:
raise error
@@ -3915,16 +4022,17 @@ def update_config_template_run_step(self):
"""Update the the easyconfig template dictionary with easyconfig.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP names"""
for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
- self.cfg.template_values[name[0]] = str(getattr(self, name[0], None))
+ self.cfg.template_values[name] = str(getattr(self, name, None))
self.cfg.generate_template_values()
def skip_step(self, step, skippable):
- """Dedice whether or not to skip the specified step."""
+ """Decide whether or not to skip the specified step."""
skip = False
force = build_option('force')
- module_only = build_option('module_only')
+ module_only = build_option('module_only') or self.cfg['module_only']
sanity_check_only = build_option('sanity_check_only')
skip_extensions = build_option('skip_extensions')
+ skip_sanity_check = build_option('skip_sanity_check')
skip_test_step = build_option('skip_test_step')
skipsteps = self.cfg['skipsteps']
@@ -3952,6 +4060,10 @@ def skip_step(self, step, skippable):
self.log.info("Skipping %s step because of sanity-check-only mode", step)
skip = True
+ elif skip_sanity_check and step == SANITYCHECK_STEP:
+ self.log.info("Skipping %s step as request via skip-sanity-check", step)
+ skip = True
+
elif skip_extensions and step == EXTENSIONS_STEP:
self.log.info("Skipping %s step as requested via skip-extensions", step)
skip = True
@@ -3962,9 +4074,9 @@ def skip_step(self, step, skippable):
else:
msg = "Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s, "
- msg += "sanity_check_only: %s, skip_extensions: %s, skip_test_step: %s)"
+ msg += "sanity_check_only: %s, skip_extensions: %s, skip_test_step: %s, skip_sanity_check: %s)"
self.log.debug(msg, step, skippable, self.skip, skipsteps, module_only, force,
- sanity_check_only, skip_extensions, skip_test_step)
+ sanity_check_only, skip_extensions, skip_test_step, skip_sanity_check)
return skip
@@ -4031,15 +4143,6 @@ def ready_step_spec(initial):
return get_step(READY_STEP, "creating build dir, resetting environment", ready_substeps, False,
initial=initial)
- source_substeps = [
- (False, lambda x: x.checksum_step),
- (True, lambda x: x.extract_step),
- ]
-
- def source_step_spec(initial):
- """Return source step specified."""
- return get_step(SOURCE_STEP, "unpacking", source_substeps, True, initial=initial)
-
install_substeps = [
(False, lambda x: x.stage_install_step),
(False, lambda x: x.make_installdir),
@@ -4053,6 +4156,7 @@ def install_step_spec(initial):
# format for step specifications: (step_name, description, list of functions, skippable)
# core steps that are part of the iterated loop
+ extract_step_spec = (SOURCE_STEP, "unpacking", [lambda x: x.extract_step], True)
patch_step_spec = (PATCH_STEP, 'patching', [lambda x: x.patch_step], True)
prepare_step_spec = (PREPARE_STEP, 'preparing', [lambda x: x.prepare_step], False)
configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True)
@@ -4062,9 +4166,9 @@ def install_step_spec(initial):
# part 1: pre-iteration + first iteration
steps_part1 = [
- (FETCH_STEP, 'fetching files', [lambda x: x.fetch_step], False),
+ (FETCH_STEP, 'fetching files', [lambda x: x.fetch_step, lambda x: x.checksum_step], False),
ready_step_spec(True),
- source_step_spec(True),
+ extract_step_spec,
patch_step_spec,
prepare_step_spec,
configure_step_spec,
@@ -4078,7 +4182,7 @@ def install_step_spec(initial):
# not all parts of all steps need to be rerun (see e.g., ready, prepare)
steps_part2 = [
ready_step_spec(False),
- source_step_spec(False),
+ extract_step_spec,
patch_step_spec,
prepare_step_spec,
configure_step_spec,
@@ -4114,7 +4218,7 @@ def run_all_steps(self, run_test_cases):
Build and install this software.
run_test_cases (bool): run tests after building (e.g.: make test)
"""
- if self.cfg['stop'] and self.cfg['stop'] == 'cfg':
+ if self.cfg['stop'] == 'cfg':
return True
steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
@@ -4168,9 +4272,15 @@ def run_all_steps(self, run_test_cases):
self.run_step(step_name, step_methods)
except RunShellCmdError as err:
err.print()
- ec_path = os.path.basename(self.cfg.path)
- error_msg = f"shell command '{err.cmd_name} ...' failed in {step_name} step for {ec_path}"
- raise EasyBuildError(error_msg)
+ error_msg = (
+ f"shell command '{err.cmd_name} ...' failed with exit code {err.exit_code} "
+ f"in {step_name} step for {os.path.basename(self.cfg.path)}"
+ )
+ try:
+ step_exit_code = EasyBuildExit[f"FAIL_{step_name.upper()}_STEP"]
+ except KeyError:
+ step_exit_code = EasyBuildExit.ERROR
+ raise EasyBuildError(error_msg, exit_code=step_exit_code) from err
finally:
if not self.dry_run:
step_duration = datetime.now() - start_time
@@ -4238,11 +4348,10 @@ def build_and_install_one(ecdict, init_env):
# restore original environment, and then sanitize it
_log.info("Resetting environment")
- run.errors_found_in_log = 0
restore_env(init_env)
sanitize_env()
- cwd = os.getcwd()
+ cwd = get_cwd()
# load easyblock
easyblock = build_option('easyblock')
@@ -4273,6 +4382,7 @@ def build_and_install_one(ecdict, init_env):
# build easyconfig
error_msg = '(no error)'
+ exit_code = None
# timing info
start_time = time.time()
try:
@@ -4311,6 +4421,10 @@ def build_and_install_one(ecdict, init_env):
except EasyBuildError as err:
error_msg = err.msg
+ try:
+ exit_code = err.exit_code
+ except AttributeError:
+ exit_code = EasyBuildExit.ERROR
result = False
ended = 'ended'
@@ -4325,7 +4439,9 @@ def build_and_install_one(ecdict, init_env):
def ensure_writable_log_dir(log_dir):
"""Make sure we can write into the log dir"""
if build_option('read_only_installdir'):
- # temporarily re-enable write permissions for copying log/easyconfig to install dir
+ # temporarily re-enable write permissions for copying log/easyconfig to install dir,
+ # ensuring that we resolve symlinks
+ log_dir = os.path.realpath(log_dir)
if os.path.exists(log_dir):
adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True)
else:
@@ -4428,11 +4544,13 @@ def ensure_writable_log_dir(log_dir):
success = True
summary = 'COMPLETED'
succ = 'successfully'
+ exit_code = EasyBuildExit.SUCCESS
else:
# build failed
success = False
summary = 'FAILED'
succ = "unsuccessfully: " + error_msg
+ exit_code = EasyBuildExit.ERROR if exit_code is None else exit_code
# cleanup logs
app.close_log()
@@ -4441,11 +4559,6 @@ def ensure_writable_log_dir(log_dir):
req_time = time2str(end_timestamp - start_timestamp)
print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent)
- # check for errors
- if run.errors_found_in_log > 0:
- _log.warning("%d possible error(s) were detected in the "
- "build logs, please verify the build.", run.errors_found_in_log)
-
if app.postmsg:
print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent)
@@ -4465,7 +4578,7 @@ def ensure_writable_log_dir(log_dir):
del app
- return (success, application_log, error_msg)
+ return (success, application_log, error_msg, exit_code)
def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir):
@@ -4475,13 +4588,13 @@ def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir):
for easyblock_class in inspect.getmro(type(easyblock_instance)):
easyblock_path = inspect.getsourcefile(easyblock_class)
# if we reach EasyBlock, Extension or ExtensionEasyBlock class, we are done
- # (Extension and ExtensionEasyblock are hardcoded to avoid a cyclical import)
+ # (Extension and ExtensionEasyBlock are hardcoded to avoid a cyclical import)
if easyblock_class.__name__ in [EasyBlock.__name__, 'Extension', 'ExtensionEasyBlock']:
break
else:
easyblock_paths.add(easyblock_path)
for easyblock_path in easyblock_paths:
- easyblock_basedir, easyblock_filename = os.path.split(easyblock_path)
+ easyblock_filename = os.path.basename(easyblock_path)
copy_file(easyblock_path, os.path.join(reprod_easyblock_dir, easyblock_filename))
_log.info("Dumped easyblock %s required for reproduction to %s", easyblock_filename, reprod_easyblock_dir)
@@ -4552,7 +4665,7 @@ def build_easyconfigs(easyconfigs, output_dir, test_results):
instance = get_easyblock_instance(ec)
apps.append(instance)
- base_dir = os.getcwd()
+ base_dir = get_cwd()
# keep track of environment right before initiating builds
# note: may be different from ORIG_OS_ENVIRON, since EasyBuild may have defined additional env vars itself by now
@@ -4612,10 +4725,7 @@ def build_easyconfigs(easyconfigs, output_dir, test_results):
class StopException(Exception):
- """
- StopException class definition.
- """
- pass
+ """Exception thrown to stop running steps"""
def inject_checksums_to_json(ecs, checksum_type):
@@ -4663,14 +4773,14 @@ def inject_checksums_to_json(ecs, checksum_type):
# actually inject new checksums or overwrite existing ones (if --force)
existing_checksums = app.get_checksums_from_json(always_read=True)
- for filename in checksums:
+ for filename, checksum in checksums.items():
if filename not in existing_checksums:
- existing_checksums[filename] = checksums[filename]
+ existing_checksums[filename] = checksum
# don't do anything if the checksum already exist and is the same
- elif checksums[filename] != existing_checksums[filename]:
+ elif checksum != existing_checksums[filename]:
if build_option('force'):
print_warning("Found existing checksums for %s, overwriting them (due to --force)..." % ec_fn)
- existing_checksums[filename] = checksums[filename]
+ existing_checksums[filename] = checksum
else:
raise EasyBuildError("Found existing checksum for %s, use --force to overwrite them" % filename)
@@ -4735,7 +4845,7 @@ def make_checksum_lines(checksums, indent_level):
# back up easyconfig file before injecting checksums
ec_backup = back_up_file(ec['spec'])
- print_msg("backup of easyconfig file saved to %s..." % ec_backup, log=_log)
+ print_msg("backup of easyconfig file saved to %s" % ec_backup, log=_log)
# compute & inject checksums for sources/patches
print_msg("injecting %s checksums for sources & patches in %s..." % (checksum_type, ec_fn), log=_log)
diff --git a/easybuild/framework/easyconfig/__init__.py b/easybuild/framework/easyconfig/__init__.py
index 6d91c028a9..b19017e023 100644
--- a/easybuild/framework/easyconfig/__init__.py
+++ b/easybuild/framework/easyconfig/__init__.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py
index 2e62580db4..4fe8c9d7b5 100644
--- a/easybuild/framework/easyconfig/constants.py
+++ b/easybuild/framework/easyconfig/constants.py
@@ -1,5 +1,5 @@
#
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py
index 1b197ab912..a365656e12 100644
--- a/easybuild/framework/easyconfig/default.py
+++ b/easybuild/framework/easyconfig/default.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -60,7 +60,7 @@
# we use a tuple here so we can sort them based on the numbers
CATEGORY_NAMES = ['BUILD', 'CUSTOM', 'DEPENDENCIES', 'EXTENSIONS', 'FILEMANAGEMENT', 'HIDDEN',
'LICENSE', 'MANDATORY', 'MODULES', 'OTHER', 'TOOLCHAIN']
-ALL_CATEGORIES = dict((name, eval(name)) for name in CATEGORY_NAMES)
+ALL_CATEGORIES = {name: eval(name) for name in CATEGORY_NAMES}
# List of tuples. Each tuple has the following format (key, [default, help text, category])
DEFAULT_CONFIG = {
@@ -108,7 +108,8 @@
BUILD],
'hidden': [False, "Install module file as 'hidden' by prefixing its version with '.'", BUILD],
'installopts': ['', 'Extra options for installation', BUILD],
- 'maxparallel': [None, 'Max degree of parallelism', BUILD],
+ 'maxparallel': [16, 'Max degree of parallelism', BUILD],
+ 'module_only': [False, 'Only generate module file', BUILD],
'parallel': [None, ('Degree of parallelism for e.g. make (default: based on the number of '
'cores, active cpuset and restrictions in ulimit)'), BUILD],
'patches': [[], "List of patches to apply", BUILD],
@@ -205,8 +206,8 @@
'moduleclass': [MODULECLASS_BASE, 'Module class to be used for this software', MODULES],
'moduleforceunload': [False, 'Force unload of all modules when loading the extension', MODULES],
'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES],
- 'module_depends_on': [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module '
- '(implies recursive unloading of modules).', MODULES],
+ 'module_depends_on': [None, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module '
+ '(implies recursive unloading of modules) [DEPRECATED]', MODULES],
'recursive_module_unload': [None, "Recursive unload of all dependencies when unloading module "
"(True/False to hard enable/disable; None implies honoring "
"the --recursive-module-unload EasyBuild configuration setting",
diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index 6e0d2aa141..4a4c4eabd6 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -46,8 +46,8 @@
import functools
import os
import re
-from contextlib import contextmanager
from collections import OrderedDict
+from contextlib import contextmanager
import easybuild.tools.filetools as filetools
from easybuild.base import fancylogger
@@ -59,11 +59,13 @@
from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION, retrieve_blocks_in_spec
from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
-from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS
-from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig
+from easybuild.framework.easyconfig.parser import ALTERNATIVE_EASYCONFIG_PARAMETERS, DEPRECATED_EASYCONFIG_PARAMETERS
+from easybuild.framework.easyconfig.parser import REPLACED_PARAMETERS, EasyConfigParser
+from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig
+from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATES, DEPRECATED_EASYCONFIG_TEMPLATES
from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict
from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning, print_msg
from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG
from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN
from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme
@@ -118,11 +120,13 @@ def handle_deprecated_or_replaced_easyconfig_parameters(ec_method):
def new_ec_method(self, key, *args, **kwargs):
"""Check whether any replace easyconfig parameters are still used"""
# map deprecated parameters to their replacements, issue deprecation warning(/error)
- if key in DEPRECATED_PARAMETERS:
+ if key in ALTERNATIVE_EASYCONFIG_PARAMETERS:
+ key = ALTERNATIVE_EASYCONFIG_PARAMETERS[key]
+ elif key in DEPRECATED_EASYCONFIG_PARAMETERS:
depr_key = key
- key, ver = DEPRECATED_PARAMETERS[depr_key]
- _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead." % (depr_key, key), ver)
- if key in REPLACED_PARAMETERS:
+ key, ver = DEPRECATED_EASYCONFIG_PARAMETERS[depr_key]
+ _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead" % (depr_key, key), ver)
+ elif key in REPLACED_PARAMETERS:
_log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0')
return ec_method(self, key, *args, **kwargs)
@@ -179,7 +183,8 @@ def triage_easyconfig_params(variables, ec):
for key in variables:
# validations are skipped, just set in the config
- if key in ec:
+ if any(key in d for d in (ec, DEPRECATED_EASYCONFIG_PARAMETERS.keys(),
+ ALTERNATIVE_EASYCONFIG_PARAMETERS.keys())):
ec_params[key] = variables[key]
_log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key]))
elif key in REPLACED_PARAMETERS:
@@ -298,7 +303,7 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False):
"""
# obtain list of all possible subtoolchains
_, all_tc_classes = search_toolchain('')
- subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes)
+ subtoolchains = {tc_class.NAME: getattr(tc_class, 'SUBTOOLCHAIN', None) for tc_class in all_tc_classes}
optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False))
composite_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1)
@@ -436,6 +441,8 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
self.path = path
self.rawtxt = read_file(path)
self.log.debug("Raw contents from supplied easyconfig file %s: %s", path, self.rawtxt)
+ if not self.rawtxt.strip():
+ raise EasyBuildError('Easyconfig file is empty')
else:
self.rawtxt = rawtxt
self.log.debug("Supplied raw easyconfig contents: %s" % self.rawtxt)
@@ -658,7 +665,8 @@ def set_keys(self, params):
with self.disable_templating():
for key in sorted(params.keys()):
# validations are skipped, just set in the config
- if key in self._config.keys():
+ if any(key in x.keys() for x in (self._config, ALTERNATIVE_EASYCONFIG_PARAMETERS,
+ DEPRECATED_EASYCONFIG_PARAMETERS)):
self[key] = params[key]
self.log.info("setting easyconfig parameter %s: value %s (type: %s)",
key, self[key], type(self[key]))
@@ -792,7 +800,7 @@ def local_var_naming(self, local_var_naming_check):
msg = "Use of %d unknown easyconfig parameters detected %s: %s\n" % (cnt, in_fn, unknown_keys_msg)
msg += "If these are just local variables please rename them to start with '%s', " % LOCAL_VAR_PREFIX
msg += "or try using --fix-deprecated-easyconfigs to do this automatically.\nFor more information, see "
- msg += "https://easybuild.readthedocs.io/en/latest/Easyconfig-files-local-variables.html ."
+ msg += "https://docs.easybuild.io/easyconfig-files-local-variables/ ."
# always log a warning if local variable that don't follow recommended naming scheme are found
self.log.warning(msg)
@@ -827,10 +835,10 @@ def check_deprecated(self, path):
if depr_msgs:
depr_msg = ', '.join(depr_msgs)
- depr_maj_ver = int(str(VERSION).split('.')[0]) + 1
+ depr_maj_ver = int(str(VERSION).split('.', maxsplit=1)[0]) + 1
depr_ver = '%s.0' % depr_maj_ver
- more_info_depr_ec = " (see also http://easybuild.readthedocs.org/en/latest/Deprecated-easyconfigs.html)"
+ more_info_depr_ec = " (see also https://docs.easybuild.io/deprecated-easyconfigs)"
self.log.deprecated(depr_msg, depr_ver, more_info=more_info_depr_ec, silent=build_option('silent'))
@@ -842,8 +850,8 @@ def validate(self, check_osdeps=True):
- check license
"""
self.log.info("Validating easyconfig")
- for attr in self.validations:
- self._validate(attr, self.validations[attr])
+ for attr, valid_values in self.validations.items():
+ self._validate(attr, valid_values)
if check_osdeps:
self.log.info("Checking OS dependencies")
@@ -899,9 +907,12 @@ def validate_os_deps(self):
not_found.append(dep)
if not_found:
- raise EasyBuildError("One or more OS dependencies were not found: %s", not_found)
- else:
- self.log.info("OS dependencies ok: %s" % self['osdependencies'])
+ raise EasyBuildError(
+ "One or more OS dependencies were not found: %s", not_found,
+ exit_code=EasyBuildExit.MISSING_SYSTEM_DEPENDENCY
+ )
+
+ self.log.info("OS dependencies ok: %s" % self['osdependencies'])
return True
@@ -963,7 +974,7 @@ def filter_hidden_deps(self):
faulty_deps = []
# obtain reference to original lists, so their elements can be changed in place
- deps = dict([(key, self.get_ref(key)) for key in ['dependencies', 'builddependencies', 'hiddendependencies']])
+ deps = {key: self.get_ref(key) for key in ('dependencies', 'builddependencies', 'hiddendependencies')}
if 'builddependencies' in self.iterate_options:
deplists = copy.deepcopy(deps['builddependencies'])
@@ -1094,15 +1105,19 @@ def filter_deps(self, deps):
return retained_deps
- def dependencies(self, build_only=False):
+ def dependencies(self, build_only=False, runtime_only=False):
"""
Returns an array of parsed dependencies (after filtering, if requested)
dependency = {'name': '', 'version': '', 'system': (False|True), 'versionsuffix': '', 'toolchain': ''}
Iterable builddependencies are flattened when not iterating.
:param build_only: only return build dependencies, discard others
+ :param runtime_only: only return runtime dependencies, discard others
"""
- deps = self.builddependencies()
+ if runtime_only:
+ deps = []
+ else:
+ deps = self.builddependencies()
if not build_only:
# use += rather than .extend to get a new list rather than updating list of build deps in place...
@@ -1207,11 +1222,11 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals
# templated values should be dumped unresolved
with self.disable_templating():
# build dict of default values
- default_values = dict([(key, DEFAULT_CONFIG[key][0]) for key in DEFAULT_CONFIG])
- default_values.update(dict([(key, self.extra_options[key][0]) for key in self.extra_options]))
+ default_values = {key: value[0] for key, value in DEFAULT_CONFIG.items()}
+ default_values.update({key: value[0] for key, value in self.extra_options.items()})
self.generate_template_values()
- templ_const = dict([(quote_py_str(const[1]), const[0]) for const in TEMPLATE_CONSTANTS])
+ templ_const = {quote_py_str(value): name for name, (value, _) in TEMPLATE_CONSTANTS.items()}
# create reverse map of templates, to inject template values where possible
# longer template values are considered first, shorter template keys get preference over longer ones
@@ -1264,7 +1279,10 @@ def _validate(self, attr, values): # private method
if values is None:
values = []
if self[attr] and self[attr] not in values:
- raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values)
+ raise EasyBuildError(
+ "%s provided '%s' is not valid: %s", attr, self[attr], values,
+ exit_code=EasyBuildExit.VALUE_ERROR
+ )
def probe_external_module_metadata(self, mod_name, existing_metadata=None):
"""
@@ -1498,7 +1516,6 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
# convert tuple to string otherwise python might complain about the formatting
self.log.debug("Parsing %s as a dependency" % str(dep))
- attr = ['name', 'version', 'versionsuffix', 'toolchain']
dependency = {
# full/short module names
'full_mod_name': None,
@@ -1554,6 +1571,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
raise EasyBuildError("Incorrect external dependency specification: %s", dep)
else:
# non-external dependency: tuple (or list) that specifies name/version(/versionsuffix(/toolchain))
+ attr = ('name', 'version', 'versionsuffix', 'toolchain')
dependency.update(dict(zip(attr, dep)))
else:
@@ -1836,7 +1854,7 @@ def get_cuda_cc_template_value(self, key):
Returns user-friendly error message in case neither are defined,
or if an unknown key is used.
"""
- if key.startswith('cuda_') and any(x[0] == key for x in TEMPLATE_NAMES_DYNAMIC):
+ if key.startswith('cuda_') and any(x == key for x in TEMPLATE_NAMES_DYNAMIC):
try:
return self.template_values[key]
except KeyError:
@@ -1878,7 +1896,9 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
else:
# if no easyblock specified, try to find if one exists
if name is None:
- name = "UNKNOWN"
+ if error_on_missing_easyblock:
+ raise EasyBuildError("No easyblock found as neither name nor easyblock were specified")
+ return None
# The following is a generic way to calculate unique class names for any funny software title
class_name = encode_class_name(name)
# modulepath will be the namespace + encoded modulename (from the classname)
@@ -1912,12 +1932,20 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
error_re = re.compile(r"No module named '?.*/?%s'?" % modname)
_log.debug("error regexp for ImportError on '%s' easyblock: %s", modname, error_re.pattern)
if error_re.match(str(err)):
+ # Missing easyblock type of error
if error_on_missing_easyblock:
- raise EasyBuildError("No software-specific easyblock '%s' found for %s", class_name, name)
- elif error_on_failed_import:
- raise EasyBuildError("Failed to import %s easyblock: %s", class_name, err)
+ raise EasyBuildError(
+ "No software-specific easyblock '%s' found for %s", class_name, name,
+ exit_code=EasyBuildExit.MISSING_EASYBLOCK
+ ) from err
else:
- _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err))
+ # Broken import
+ if error_on_failed_import:
+ raise EasyBuildError(
+ "Failed to import %s easyblock: %s", class_name, err,
+ exit_code=EasyBuildExit.EASYBLOCK_ERROR
+ ) from err
+ _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err))
if cls is not None:
_log.info("Successfully obtained class '%s' for easyblock '%s' (software name '%s')",
@@ -1931,7 +1959,10 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
# simply reraise rather than wrapping it into another error
raise err
except Exception as err:
- raise EasyBuildError("Failed to obtain class for %s easyblock (not available?): %s", easyblock, err)
+ raise EasyBuildError(
+ "Failed to obtain class for %s easyblock (not available?): %s", easyblock, err,
+ exit_code=EasyBuildExit.EASYBLOCK_ERROR
+ )
def get_module_path(name, generic=None, decode=True):
@@ -1989,12 +2020,41 @@ def resolve_template(value, tmpl_dict):
# '%(name)s' -> '%(name)s'
# '%%(name)s' -> '%%(name)s'
if '%' in value:
+ raw_value = value
value = re.sub(re.compile(r'(%)(?!%*\(\w+\)s)'), r'\1\1', value)
try:
value = value % tmpl_dict
except KeyError:
- _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict)
+ # check if any alternative and/or deprecated templates resolve
+ try:
+ orig_value = value
+ # map old templates to new values for alternative and deprecated templates
+ alt_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, new_tmpl) in
+ ALTERNATIVE_EASYCONFIG_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()}
+ alt_map2 = {new_tmpl: tmpl_dict[old_tmpl] for (old_tmpl, new_tmpl) in
+ ALTERNATIVE_EASYCONFIG_TEMPLATES.items() if old_tmpl in tmpl_dict.keys()}
+ depr_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, (new_tmpl, ver)) in
+ DEPRECATED_EASYCONFIG_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()}
+
+ # try templating with alternative and deprecated templates included
+ value = value % {**tmpl_dict, **alt_map, **alt_map2, **depr_map}
+
+ for old_tmpl, val in depr_map.items():
+ # check which deprecated templates were replaced, and issue deprecation warnings
+ if old_tmpl in orig_value and val in value:
+ new_tmpl, ver = DEPRECATED_EASYCONFIG_TEMPLATES[old_tmpl]
+ _log.deprecated(f"Easyconfig template '{old_tmpl}' is deprecated, use '{new_tmpl}' instead",
+ ver)
+ except KeyError:
+ _log.warning(f"Unable to resolve template value {value} with dict {tmpl_dict}")
+ value = raw_value # Undo "%"-escaping
+
+ for key in tmpl_dict:
+ if key in DEPRECATED_EASYCONFIG_TEMPLATES:
+ new_key, ver = DEPRECATED_EASYCONFIG_TEMPLATES[key]
+ _log.deprecated(f"Easyconfig template '{key}' is deprecated, use '{new_key}' instead", ver)
+
else:
# this block deals with references to objects and returns other references
# for reading this is ok, but for self['x'] = {}
@@ -2012,7 +2072,7 @@ def resolve_template(value, tmpl_dict):
elif isinstance(value, tuple):
value = tuple(resolve_template(list(value), tmpl_dict))
elif isinstance(value, dict):
- value = dict((resolve_template(k, tmpl_dict), resolve_template(v, tmpl_dict)) for k, v in value.items())
+ value = {resolve_template(k, tmpl_dict): resolve_template(v, tmpl_dict) for k, v in value.items()}
return value
@@ -2047,7 +2107,11 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False,
try:
ec = EasyConfig(spec, build_specs=build_specs, validate=validate, hidden=hidden)
except EasyBuildError as err:
- raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg)
+ try:
+ exit_code = err.exit_code
+ except AttributeError:
+ exit_code = EasyBuildExit.EASYCONFIG_ERROR
+ raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg, exit_code=exit_code)
name = ec['name']
@@ -2219,7 +2283,7 @@ def verify_easyconfig_filename(path, specs, parsed_ec=None):
for ec in ecs:
found_fullver = det_full_ec_version(ec['ec'])
if ec['ec']['name'] != specs['name'] or found_fullver != fullver:
- subspec = dict((key, specs[key]) for key in ['name', 'toolchain', 'version', 'versionsuffix'])
+ subspec = {key: specs[key] for key in ('name', 'toolchain', 'version', 'versionsuffix')}
error_msg = "Contents of %s does not match with filename" % path
error_msg += "; expected filename based on contents: %s-%s.eb" % (ec['ec']['name'], found_fullver)
error_msg += "; expected (relevant) parameters based on filename %s: %s" % (os.path.basename(path), subspec)
diff --git a/easybuild/framework/easyconfig/format/__init__.py b/easybuild/framework/easyconfig/format/__init__.py
index b2c084e273..c2a81766ad 100644
--- a/easybuild/framework/easyconfig/format/__init__.py
+++ b/easybuild/framework/easyconfig/format/__init__.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyconfig/format/convert.py b/easybuild/framework/easyconfig/format/convert.py
index 726acd3e6c..82613d42e0 100644
--- a/easybuild/framework/easyconfig/format/convert.py
+++ b/easybuild/framework/easyconfig/format/convert.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py
index da8bb97c89..1ae367c557 100644
--- a/easybuild/framework/easyconfig/format/format.py
+++ b/easybuild/framework/easyconfig/format/format.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -433,7 +433,7 @@ def _squash(self, vt_tuple, processed, sanity):
# walk over dictionary of parsed sections, and check for marker conflicts (using .add())
for key, value in processed.items():
if isinstance(value, NestedDict):
- tmp = self._squash_netsed_dict(key, value, squashed, sanity, vt_tuple)
+ tmp = self._squash_nested_dict(key, value, squashed, sanity, vt_tuple)
res_sections.update(tmp)
elif key in self.VERSION_OPERATOR_VALUE_TYPES:
self.log.debug("Found VERSION_OPERATOR_VALUE_TYPES entry (%s)" % key)
@@ -453,7 +453,7 @@ def _squash(self, vt_tuple, processed, sanity):
(processed, squashed.versions, squashed.result))
return squashed
- def _squash_netsed_dict(self, key, nested_dict, squashed, sanity, vt_tuple):
+ def _squash_nested_dict(self, key, nested_dict, squashed, sanity, vt_tuple):
"""
Squash NestedDict instance, returns dict with already squashed data
from possible higher sections
diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py
index 994fc93fed..a4a6041c39 100644
--- a/easybuild/framework/easyconfig/format/one.py
+++ b/easybuild/framework/easyconfig/format/one.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py
index 37e14b529e..c52a459322 100644
--- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py
+++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -39,7 +39,8 @@
from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
from easybuild.framework.easyconfig.format.format import get_format_version, EasyConfigFormat
from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
-from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS
+from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATE_CONSTANTS
+from easybuild.framework.easyconfig.templates import DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS, TEMPLATE_CONSTANTS
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.configobj import ConfigObj
from easybuild.tools.systemtools import get_shared_lib_ext
@@ -51,9 +52,9 @@
def build_easyconfig_constants_dict():
"""Make a dictionary with all constants that can be used"""
all_consts = [
- ('TEMPLATE_CONSTANTS', dict([(x[0], x[1]) for x in TEMPLATE_CONSTANTS])),
- ('EASYCONFIG_CONSTANTS', dict([(key, val[0]) for key, val in EASYCONFIG_CONSTANTS.items()])),
- ('EASYCONFIG_LICENSES', dict([(klass().name, name) for name, klass in EASYCONFIG_LICENSES_DICT.items()])),
+ ('TEMPLATE_CONSTANTS', {name: value for name, (value, _) in TEMPLATE_CONSTANTS.items()}),
+ ('EASYCONFIG_CONSTANTS', {name: value for name, (value, _) in EASYCONFIG_CONSTANTS.items()}),
+ ('EASYCONFIG_LICENSES', {klass().name: name for name, klass in EASYCONFIG_LICENSES_DICT.items()}),
]
err = []
const_dict = {}
@@ -86,6 +87,58 @@ def build_easyconfig_variables_dict():
return vars_dict
+def handle_deprecated_constants(method):
+ """Decorator to handle deprecated easyconfig template constants"""
+ def wrapper(self, key, *args, **kwargs):
+ """Check whether any deprecated constants are used"""
+ alternative = ALTERNATIVE_EASYCONFIG_TEMPLATE_CONSTANTS
+ deprecated = DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS
+ if key in alternative:
+ key = alternative[key]
+ elif key in deprecated:
+ depr_key = key
+ key, ver = deprecated[depr_key]
+ _log.deprecated(f"Easyconfig template constant '{depr_key}' is deprecated, use '{key}' instead", ver)
+ return method(self, key, *args, **kwargs)
+ return wrapper
+
+
+class DeprecatedDict(dict):
+ """Custom dictionary that handles deprecated easyconfig template constants gracefully"""
+
+ def __init__(self, *args, **kwargs):
+ self.clear()
+ self.update(*args, **kwargs)
+
+ @handle_deprecated_constants
+ def __contains__(self, key):
+ return super().__contains__(key)
+
+ @handle_deprecated_constants
+ def __delitem__(self, key):
+ return super().__delitem__(key)
+
+ @handle_deprecated_constants
+ def __getitem__(self, key):
+ return super().__getitem__(key)
+
+ @handle_deprecated_constants
+ def __setitem__(self, key, value):
+ return super().__setitem__(key, value)
+
+ def update(self, *args, **kwargs):
+ if args:
+ if isinstance(args[0], dict):
+ for key, value in args[0].items():
+ self.__setitem__(key, value)
+ else:
+ for key, value in args[0]:
+ self.__setitem__(key, value)
+
+ for key, value in kwargs.items():
+ self.__setitem__(key, value)
+
+
class EasyConfigFormatConfigObj(EasyConfigFormat):
"""
Extended EasyConfig format, with support for a header and sections that are actually parsed (as opposed to exec'ed).
@@ -176,7 +229,7 @@ def parse_header(self, header):
def parse_pyheader(self, pyheader):
"""Parse the python header, assign to docstring and cfg"""
- global_vars = self.pyheader_env()
+ global_vars = DeprecatedDict(self.pyheader_env())
self.log.debug("pyheader initial global_vars %s", global_vars)
self.log.debug("pyheader text being exec'ed: %s", pyheader)
diff --git a/easybuild/framework/easyconfig/format/two.py b/easybuild/framework/easyconfig/format/two.py
index cec3df3636..652b1c9b80 100644
--- a/easybuild/framework/easyconfig/format/two.py
+++ b/easybuild/framework/easyconfig/format/two.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyconfig/format/version.py b/easybuild/framework/easyconfig/format/version.py
index c9f517122b..e48e6fc8e1 100644
--- a/easybuild/framework/easyconfig/format/version.py
+++ b/easybuild/framework/easyconfig/format/version.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -45,7 +45,16 @@
class EasyVersion(LooseVersion):
- """Exact LooseVersion. No modifications needed (yet)"""
+ """Represent a version"""
+
+ def __init__(self, vstring, is_default=False):
+ super().__init__(vstring)
+ self._is_default = is_default
+
+ @property
+ def is_default(self):
+ """Return whether this is the default version used when no explicit version is specified"""
+ return self._is_default
def __len__(self):
"""Determine length of this EasyVersion instance."""
@@ -68,13 +77,13 @@ class VersionOperator(object):
'<': op.lt,
'<=': op.le,
}
- REVERSE_OPERATOR_MAP = dict([(v, k) for k, v in OPERATOR_MAP.items()])
+ REVERSE_OPERATOR_MAP = {v: k for k, v in OPERATOR_MAP.items()}
INCLUDE_OPERATORS = ['==', '>=', '<='] # these operators *include* the (version) boundary
ORDERED_OPERATORS = ['==', '>', '>=', '<', '<='] # ordering by strictness
OPERATOR_FAMILIES = [['>', '>='], ['<', '<=']] # similar operators
# default version and operator when version is undefined
- DEFAULT_UNDEFINED_VERSION = EasyVersion('0.0.0')
+ DEFAULT_UNDEFINED_VERSION = EasyVersion('0.0', is_default=True)
DEFAULT_UNDEFINED_VERSION_OPERATOR = OPERATOR_MAP['>']
# default operator when operator is undefined (but version is)
DEFAULT_UNDEFINED_OPERATOR = OPERATOR_MAP['==']
@@ -256,7 +265,7 @@ def _convert_operator(self, operator_str, version=None):
"""Return the operator"""
operator = None
if operator_str is None:
- if version == self.DEFAULT_UNDEFINED_VERSION or version is None:
+ if version is None or version.is_default:
operator = self.DEFAULT_UNDEFINED_VERSION_OPERATOR
else:
operator = self.DEFAULT_UNDEFINED_OPERATOR
diff --git a/easybuild/framework/easyconfig/licenses.py b/easybuild/framework/easyconfig/licenses.py
index e8eb1ee3bc..e6286d54e3 100644
--- a/easybuild/framework/easyconfig/licenses.py
+++ b/easybuild/framework/easyconfig/licenses.py
@@ -1,5 +1,5 @@
#
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py
index dd53f48890..7ebdff4873 100644
--- a/easybuild/framework/easyconfig/parser.py
+++ b/easybuild/framework/easyconfig/parser.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -42,8 +42,66 @@
from easybuild.tools.filetools import read_file, write_file
+# alternative easyconfig parameters, and their non-deprecated equivalents
+ALTERNATIVE_EASYCONFIG_PARAMETERS = {
+ # : ,
+ 'build_deps': 'builddependencies',
+ 'build_in_install_dir': 'buildininstalldir',
+ 'build_opts': 'buildopts',
+ 'build_stats': 'buildstats',
+ 'clean_up_old_build': 'cleanupoldbuild',
+ 'clean_up_old_install': 'cleanupoldinstall',
+ 'configure_opts': 'configopts',
+ 'deps': 'dependencies',
+ 'doc_paths': 'docpaths',
+ 'doc_urls': 'docurls',
+ 'do_not_create_install_dir': 'dontcreateinstalldir',
+ 'exts_class_map': 'exts_classmap',
+ 'exts_default_class': 'exts_defaultclass',
+ 'exts_default_opts': 'exts_default_options',
+ 'hidden_deps': 'hiddendependencies',
+ 'include_modulepath_exts': 'include_modpath_extensions',
+ 'install_opts': 'installopts',
+ 'keep_previous_install': 'keeppreviousinstall',
+ 'keep_symlinks': 'keepsymlinks',
+ 'max_parallel': 'maxparallel',
+ 'env_mod_aliases': 'modaliases',
+ 'env_mod_alt_soft_name': 'modaltsoftname',
+ 'modulepath_prepend_paths': 'moddependpaths',
+ 'env_mod_extra_paths_append': 'modextrapaths_append',
+ 'env_mod_extra_paths': 'modextrapaths',
+ 'env_mod_extra_vars': 'modextravars',
+ 'env_mod_load_msg': 'modloadmsg',
+ 'env_mod_lua_footer': 'modluafooter',
+ 'env_mod_tcl_footer': 'modtclfooter',
+ 'env_mod_category': 'moduleclass',
+ 'env_mod_depends_on': 'module_depends_on',
+ 'env_mod_force_unload': 'moduleforceunload',
+ 'env_mod_load_no_conflict': 'moduleloadnoconflict',
+ 'env_mod_unload_msg': 'modunloadmsg',
+ 'only_toolchain_env_mod': 'onlytcmod',
+ 'os_deps': 'osdependencies',
+ 'post_install_cmds': 'postinstallcmds',
+ 'post_install_msgs': 'postinstallmsgs',
+ 'post_install_patches': 'postinstallpatches',
+ 'pre_build_opts': 'prebuildopts',
+ 'pre_configure_opts': 'preconfigopts',
+ 'pre_install_opts': 'preinstallopts',
+ 'pre_test_opts': 'pretestopts',
+ 'recursive_env_mod_unload': 'recursive_module_unload',
+ 'run_test': 'runtest',
+ 'sanity_check_cmds': 'sanity_check_commands',
+ 'skip_fortran_mod_files_sanity_check': 'skip_mod_files_sanity_check',
+ 'skip_steps': 'skipsteps',
+ 'test_opts': 'testopts',
+ 'toolchain_opts': 'toolchainopts',
+ 'unpack_opts': 'unpack_options',
+ 'version_prefix': 'versionprefix',
+ 'version_suffix': 'versionsuffix',
+}
+
# deprecated easyconfig parameters, and their replacements
-DEPRECATED_PARAMETERS = {
+DEPRECATED_EASYCONFIG_PARAMETERS = {
# : (, ),
}
diff --git a/easybuild/framework/easyconfig/style.py b/easybuild/framework/easyconfig/style.py
index 10139118e9..9e264d4f3d 100644
--- a/easybuild/framework/easyconfig/style.py
+++ b/easybuild/framework/easyconfig/style.py
@@ -1,5 +1,5 @@
##
-# Copyright 2016-2023 Ghent University
+# Copyright 2016-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -31,7 +31,6 @@
* Ward Poelmans (Ghent University)
"""
import re
-import sys
from importlib import reload
from easybuild.base import fancylogger
@@ -43,12 +42,7 @@
import pycodestyle
from pycodestyle import StyleGuide, register_check, trailing_whitespace
except ImportError:
- try:
- # fallback to importing from 'pep8', which was renamed to pycodestyle in 2016
- import pep8
- from pep8 import StyleGuide, register_check, trailing_whitespace
- except ImportError:
- pass
+ pass
_log = fancylogger.getLogger('easyconfig.style', fname=False)
@@ -106,7 +100,7 @@ def _eb_check_trailing_whitespace(physical_line, lines, line_number, checker_sta
return result
-@only_if_module_is_available(('pycodestyle', 'pep8'))
+@only_if_module_is_available('pycodestyle')
def check_easyconfigs_style(easyconfigs, verbose=False):
"""
Check the given list of easyconfigs for style
@@ -114,12 +108,7 @@ def check_easyconfigs_style(easyconfigs, verbose=False):
:param verbose: print our statistics and be verbose about the errors and warning
:return: the number of warnings and errors
"""
- # importing autopep8 changes some pep8 functions.
- # We reload it to be sure to get the real pep8 functions.
- if 'pycodestyle' in sys.modules:
- reload(pycodestyle)
- else:
- reload(pep8)
+ reload(pycodestyle)
# register the extra checks before using pep8:
# any function in this module starting with `_eb_check_` will be used.
@@ -137,6 +126,7 @@ def check_easyconfigs_style(easyconfigs, verbose=False):
# note that W291 has been replaced by our custom W299
options.ignore = (
'W291', # replaced by W299
+ 'E741', # 'l' is considered an ambiguous name, but we use it often for 'lib'
)
options.verbose = int(verbose)
diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py
index 0d8a4d5cf3..4486a9f36e 100644
--- a/easybuild/framework/easyconfig/templates.py
+++ b/easybuild/framework/easyconfig/templates.py
@@ -1,5 +1,5 @@
#
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -45,15 +45,15 @@
_log = fancylogger.getLogger('easyconfig.templates', fname=False)
# derived from easyconfig, but not from ._config directly
-TEMPLATE_NAMES_EASYCONFIG = [
- ('module_name', "Module name"),
- ('nameletter', "First letter of software name"),
- ('toolchain_name', "Toolchain name"),
- ('toolchain_version', "Toolchain version"),
- ('version_major_minor', "Major.Minor version"),
- ('version_major', "Major version"),
- ('version_minor', "Minor version"),
-]
+TEMPLATE_NAMES_EASYCONFIG = {
+ 'module_name': 'Module name',
+ 'nameletter': 'First letter of software name',
+ 'toolchain_name': 'Toolchain name',
+ 'toolchain_version': 'Toolchain version',
+ 'version_major_minor': "Major.Minor version",
+ 'version_major': 'Major version',
+ 'version_minor': 'Minor version',
+}
# derived from EasyConfig._config
TEMPLATE_NAMES_CONFIG = [
'bitbucket_account',
@@ -71,103 +71,202 @@
'nameletter',
]
# values taken from the EasyBlock before each step
-TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = [
- ('builddir', "Build directory"),
- ('installdir', "Installation directory"),
- ('start_dir', "Directory in which the build process begins"),
-]
+TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = {
+ 'builddir': 'Build directory',
+ 'installdir': 'Installation directory',
+ 'start_dir': 'Directory in which the build process begins',
+}
# software names for which to define ver, majver and shortver templates
-TEMPLATE_SOFTWARE_VERSIONS = [
- # software name, prefix for *ver, *majver and *shortver
- ('CUDA', 'cuda'),
- ('CUDAcore', 'cuda'),
- ('Java', 'java'),
- ('Perl', 'perl'),
- ('Python', 'py'),
- ('R', 'r'),
-]
+TEMPLATE_SOFTWARE_VERSIONS = {
+ # software name -> prefix for *ver, *majver and *shortver
+ 'CUDA': 'cuda',
+ 'CUDAcore': 'cuda',
+ 'Java': 'java',
+ 'Perl': 'perl',
+ 'Python': 'py',
+ 'R': 'r',
+}
# template values which are only generated dynamically
-TEMPLATE_NAMES_DYNAMIC = [
- ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"),
- ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include"
- "as specify by the --sysroot configuration option"),
- ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"),
- ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via "
- "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"),
- ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"),
- ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"),
- ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"),
- ('cuda_int_comma_sep', "Comma-separated list of integer CUDA compute capabilities"),
- ('cuda_int_space_sep', "Space-separated list of integer CUDA compute capabilities"),
- ('cuda_int_semicolon_sep', "Semicolon-separated list of integer CUDA compute capabilities"),
- ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"),
- ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"),
-]
+TEMPLATE_NAMES_DYNAMIC = {
+ 'arch': 'System architecture (e.g. x86_64, aarch64, ppc64le, ...)',
+ 'cuda_compute_capabilities': "Comma-separated list of CUDA compute capabilities, as specified via "
+ "--cuda-compute-capabilities configuration option or "
+ "via cuda_compute_capabilities easyconfig parameter",
+ 'cuda_cc_cmake': 'List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+',
+ 'cuda_cc_space_sep': 'Space-separated list of CUDA compute capabilities',
+ 'cuda_cc_space_sep_no_period':
+ "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90').",
+ 'cuda_cc_semicolon_sep': 'Semicolon-separated list of CUDA compute capabilities',
+ 'cuda_int_comma_sep': 'Comma-separated list of integer CUDA compute capabilities',
+ 'cuda_int_space_sep': 'Space-separated list of integer CUDA compute capabilities',
+ 'cuda_int_semicolon_sep': 'Semicolon-separated list of integer CUDA compute capabilities',
+ 'cuda_sm_comma_sep': 'Comma-separated list of sm_* values that correspond with CUDA compute capabilities',
+ 'cuda_sm_space_sep': 'Space-separated list of sm_* values that correspond with CUDA compute capabilities',
+ 'mpi_cmd_prefix': 'Prefix command for running MPI programs (with default number of ranks)',
+ # can't be a boolean (True/False), must be a string value since it's a string template
+ 'rpath_enabled': "String value indicating whether or not RPATH linking is used ('true' or 'false')",
+ 'software_commit': "Git commit id to use for the software as specified by --software-commit command line option",
+ 'sysroot': "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include"
+ "as specify by the --sysroot configuration option",
+
+}
# constant templates that can be used in easyconfigs
-TEMPLATE_CONSTANTS = [
+# Entry: constant -> (value, doc)
+TEMPLATE_CONSTANTS = {
# source url constants
- ('APACHE_SOURCE', 'https://archive.apache.org/dist/%(namelower)s',
- 'apache.org source url'),
- ('BITBUCKET_SOURCE', 'https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get',
- 'bitbucket.org source url (namelower is used if bitbucket_account easyconfig parameter is not specified)'),
- ('BITBUCKET_DOWNLOADS', 'https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads',
- 'bitbucket.org downloads url (namelower is used if bitbucket_account easyconfig parameter is not specified)'),
- ('CRAN_SOURCE', 'https://cran.r-project.org/src/contrib',
- 'CRAN (contrib) source url'),
- ('FTPGNOME_SOURCE', 'https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s',
- 'http download for gnome ftp server'),
- ('GITHUB_SOURCE', 'https://github.com/%(github_account)s/%(name)s/archive',
- 'GitHub source URL (if github_account easyconfig parameter is not specified, namelower is used in its place)'),
- ('GITHUB_LOWER_SOURCE', 'https://github.com/%(github_account)s/%(namelower)s/archive',
- 'GitHub source URL with lowercase name (if github_account easyconfig '
- 'parameter is not specified, namelower is used in its place)'),
- ('GITHUB_RELEASE', 'https://github.com/%(github_account)s/%(name)s/releases/download/v%(version)s',
- 'GitHub release URL (if github_account easyconfig parameter is not specified, namelower is used in its place)'),
- ('GITHUB_LOWER_RELEASE', 'https://github.com/%(github_account)s/%(namelower)s/releases/download/v%(version)s',
- 'GitHub release URL with lowercase name (if github_account easyconfig '
- 'parameter is not specified, namelower is used in its place)'),
- ('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s',
- 'download.savannah.gnu.org source url'),
- ('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s',
- 'gnu.org source url'),
- ('GOOGLECODE_SOURCE', 'http://%(namelower)s.googlecode.com/files',
- 'googlecode.com source url'),
- ('LAUNCHPAD_SOURCE', 'https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/',
- 'launchpad.net source url'),
- ('PYPI_SOURCE', 'https://pypi.python.org/packages/source/%(nameletter)s/%(name)s',
- 'pypi source url'), # e.g., Cython, Sphinx
- ('PYPI_LOWER_SOURCE', 'https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s',
- 'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ
- ('R_SOURCE', 'https://cran.r-project.org/src/base/R-%(version_major)s',
- 'cran.r-project.org (base) source url'),
- ('SOURCEFORGE_SOURCE', 'https://download.sourceforge.net/%(namelower)s',
- 'sourceforge.net source url'),
- ('XORG_DATA_SOURCE', 'https://xorg.freedesktop.org/archive/individual/data/',
- 'xorg data source url'),
- ('XORG_LIB_SOURCE', 'https://xorg.freedesktop.org/archive/individual/lib/',
- 'xorg lib source url'),
- ('XORG_PROTO_SOURCE', 'https://xorg.freedesktop.org/archive/individual/proto/',
- 'xorg proto source url'),
- ('XORG_UTIL_SOURCE', 'https://xorg.freedesktop.org/archive/individual/util/',
- 'xorg util source url'),
- ('XORG_XCB_SOURCE', 'https://xorg.freedesktop.org/archive/individual/xcb/',
- 'xorg xcb source url'),
+ 'APACHE_SOURCE': ('https://archive.apache.org/dist/%(namelower)s',
+ 'apache.org source url'),
+ 'BITBUCKET_SOURCE': ('https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get',
+ 'bitbucket.org source url '
+ '(namelower is used if bitbucket_account easyconfig parameter is not specified)'),
+ 'BITBUCKET_DOWNLOADS': ('https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads',
+ 'bitbucket.org downloads url '
+ '(namelower is used if bitbucket_account easyconfig parameter is not specified)'),
+ 'CRAN_SOURCE': ('https://cran.r-project.org/src/contrib',
+ 'CRAN (contrib) source url'),
+ 'FTPGNOME_SOURCE': ('https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s',
+ 'http download for gnome ftp server'),
+ 'GITHUB_SOURCE': ('https://github.com/%(github_account)s/%(name)s/archive',
+ 'GitHub source URL '
+ '(namelower is used if github_account easyconfig parameter is not specified)'),
+ 'GITHUB_LOWER_SOURCE': ('https://github.com/%(github_account)s/%(namelower)s/archive',
+ 'GitHub source URL with lowercase name '
+ '(namelower is used if github_account easyconfig parameter is not specified)'),
+ 'GITHUB_RELEASE': ('https://github.com/%(github_account)s/%(name)s/releases/download/v%(version)s',
+ 'GitHub release URL '
+ '(namelower is use if github_account easyconfig parameter is not specified)'),
+ 'GITHUB_LOWER_RELEASE': ('https://github.com/%(github_account)s/%(namelower)s/releases/download/v%(version)s',
+ 'GitHub release URL with lowercase name (if github_account easyconfig '
+ 'parameter is not specified, namelower is used in its place)'),
+ 'GNU_SAVANNAH_SOURCE': ('https://download-mirror.savannah.gnu.org/releases/%(namelower)s',
+ 'download.savannah.gnu.org source url'),
+ 'GNU_SOURCE': ('https://ftpmirror.gnu.org/gnu/%(namelower)s',
+ 'gnu.org source url (ftp mirror)'),
+ 'GNU_FTP_SOURCE': ('https://ftp.gnu.org/gnu/%(namelower)s',
+ 'gnu.org source url (main ftp)'),
+ 'GOOGLECODE_SOURCE': ('http://%(namelower)s.googlecode.com/files',
+ 'googlecode.com source url'),
+ 'LAUNCHPAD_SOURCE': ('https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/',
+ 'launchpad.net source url'),
+ 'PYPI_SOURCE': ('https://pypi.python.org/packages/source/%(nameletter)s/%(name)s',
+ 'pypi source url'), # e.g., Cython, Sphinx
+ 'PYPI_LOWER_SOURCE': ('https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s',
+ 'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ
+ 'R_SOURCE': ('https://cran.r-project.org/src/base/R-%(version_major)s',
+ 'cran.r-project.org (base) source url'),
+ 'SOURCEFORGE_SOURCE': ('https://download.sourceforge.net/%(namelower)s',
+ 'sourceforge.net source url'),
+ 'XORG_DATA_SOURCE': ('https://xorg.freedesktop.org/archive/individual/data/',
+ 'xorg data source url'),
+ 'XORG_LIB_SOURCE': ('https://xorg.freedesktop.org/archive/individual/lib/',
+ 'xorg lib source url'),
+ 'XORG_PROTO_SOURCE': ('https://xorg.freedesktop.org/archive/individual/proto/',
+ 'xorg proto source url'),
+ 'XORG_UTIL_SOURCE': ('https://xorg.freedesktop.org/archive/individual/util/',
+ 'xorg util source url'),
+ 'XORG_XCB_SOURCE': ('https://xorg.freedesktop.org/archive/individual/xcb/',
+ 'xorg xcb source url'),
# TODO, not urgent, yet nice to have:
# CPAN_SOURCE GNOME KDE_I18N XCONTRIB DEBIAN KDE GENTOO TEX_CTAN MOZILLA_ALL
# other constants
- ('SHLIB_EXT', get_shared_lib_ext(), 'extension for shared libraries'),
-]
-
-extensions = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z']
-for ext in extensions:
+ 'SHLIB_EXT': (get_shared_lib_ext(), 'extension for shared libraries'),
+}
+
+# alternative templates, and their equivalents
+ALTERNATIVE_EASYCONFIG_TEMPLATES = {
+ # : ,
+ 'build_dir': 'builddir',
+ 'cuda_cc_comma_sep': 'cuda_compute_capabilities',
+ 'cuda_maj_ver': 'cudamajver',
+ 'cuda_short_ver': 'cudashortver',
+ 'cuda_ver': 'cudaver',
+ 'install_dir': 'installdir',
+ 'java_maj_ver': 'javamajver',
+ 'java_short_ver': 'javashortver',
+ 'java_ver': 'javaver',
+ 'name_letter_lower': 'nameletterlower',
+ 'name_letter': 'nameletter',
+ 'name_lower': 'namelower',
+ 'perl_maj_ver': 'perlmajver',
+ 'perl_short_ver': 'perlshortver',
+ 'perl_ver': 'perlver',
+ 'py_maj_ver': 'pymajver',
+ 'py_short_ver': 'pyshortver',
+ 'py_ver': 'pyver',
+ 'r_maj_ver': 'rmajver',
+ 'r_short_ver': 'rshortver',
+ 'r_ver': 'rver',
+ 'toolchain_ver': 'toolchain_version',
+ 'ver_maj_min': 'version_major_minor',
+ 'ver_maj': 'version_major',
+ 'ver_min': 'version_minor',
+ 'version_prefix': 'versionprefix',
+ 'version_suffix': 'versionsuffix',
+}
+
+# deprecated templates, and their replacements
+DEPRECATED_EASYCONFIG_TEMPLATES = {
+ # : (, ),
+}
+
+# alternative template constants, and their equivalents
+ALTERNATIVE_EASYCONFIG_TEMPLATE_CONSTANTS = {
+ # : ,
+ 'APACHE_URL': 'APACHE_SOURCE',
+ 'BITBUCKET_GET_URL': 'BITBUCKET_SOURCE',
+ 'BITBUCKET_DOWNLOADS_URL': 'BITBUCKET_DOWNLOADS',
+ 'CRAN_URL': 'CRAN_SOURCE',
+ 'FTP_GNOME_URL': 'FTPGNOME_SOURCE',
+ 'GITHUB_URL': 'GITHUB_SOURCE',
+ 'GITHUB_URL_LOWER': 'GITHUB_LOWER_SOURCE',
+ 'GITHUB_RELEASE_URL': 'GITHUB_RELEASE',
+ 'GITHUB_RELEASE_URL_LOWER': 'GITHUB_LOWER_RELEASE',
+ 'GNU_SAVANNAH_URL': 'GNU_SAVANNAH_SOURCE',
+ 'GNU_FTP_URL': 'GNU_FTP_SOURCE',
+ 'GNU_URL': 'GNU_SOURCE',
+ 'GOOGLECODE_URL': 'GOOGLECODE_SOURCE',
+ 'LAUNCHPAD_URL': 'LAUNCHPAD_SOURCE',
+ 'PYPI_URL': 'PYPI_SOURCE',
+ 'PYPI_URL_LOWER': 'PYPI_LOWER_SOURCE',
+ 'R_URL': 'R_SOURCE',
+ 'SOURCEFORGE_URL': 'SOURCEFORGE_SOURCE',
+ 'XORG_DATA_URL': 'XORG_DATA_SOURCE',
+ 'XORG_LIB_URL': 'XORG_LIB_SOURCE',
+ 'XORG_PROTO_URL': 'XORG_PROTO_SOURCE',
+ 'XORG_UTIL_URL': 'XORG_UTIL_SOURCE',
+ 'XORG_XCB_URL': 'XORG_XCB_SOURCE',
+ 'SOURCE_LOWER_TAR_GZ': 'SOURCELOWER_TAR_GZ',
+ 'SOURCE_LOWER_TAR_XZ': 'SOURCELOWER_TAR_XZ',
+ 'SOURCE_LOWER_TAR_BZ2': 'SOURCELOWER_TAR_BZ2',
+ 'SOURCE_LOWER_TGZ': 'SOURCELOWER_TGZ',
+ 'SOURCE_LOWER_TXZ': 'SOURCELOWER_TXZ',
+ 'SOURCE_LOWER_TBZ2': 'SOURCELOWER_TBZ2',
+ 'SOURCE_LOWER_TB2': 'SOURCELOWER_TB2',
+ 'SOURCE_LOWER_GTGZ': 'SOURCELOWER_GTGZ',
+ 'SOURCE_LOWER_ZIP': 'SOURCELOWER_ZIP',
+ 'SOURCE_LOWER_TAR': 'SOURCELOWER_TAR',
+ 'SOURCE_LOWER_XZ': 'SOURCELOWER_XZ',
+ 'SOURCE_LOWER_TAR_Z': 'SOURCELOWER_TAR_Z',
+ 'SOURCE_LOWER_WHL': 'SOURCELOWER_WHL',
+ 'SOURCE_LOWER_PY2_WHL': 'SOURCELOWER_PY2_WHL',
+ 'SOURCE_LOWER_PY3_WHL': 'SOURCELOWER_PY3_WHL',
+}
+
+# deprecated template constants, and their replacements
+DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS = {
+ # : (, ),
+}
+
+EXTENSIONS = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z']
+for ext in EXTENSIONS:
suffix = ext.replace('.', '_').upper()
- TEMPLATE_CONSTANTS += [
- ('SOURCE_%s' % suffix, '%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext),
- ('SOURCELOWER_%s' % suffix, '%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext),
- ]
+ TEMPLATE_CONSTANTS.update({
+ 'SOURCE_%s' % suffix: ('%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext),
+ 'SOURCELOWER_%s' % suffix: ('%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext),
+ })
for pyver in ('py2.py3', 'py2', 'py3'):
if pyver == 'py2.py3':
desc = 'Python 2 & Python 3'
@@ -175,12 +274,12 @@
else:
desc = 'Python ' + pyver[-1]
name_infix = pyver.upper() + '_'
- TEMPLATE_CONSTANTS += [
- ('SOURCE_%sWHL' % name_infix, '%%(name)s-%%(version)s-%s-none-any.whl' % pyver,
- 'Generic (non-compiled) %s wheel package' % desc),
- ('SOURCELOWER_%sWHL' % name_infix, '%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver,
- 'Generic (non-compiled) %s wheel package with lowercase name' % desc),
- ]
+ TEMPLATE_CONSTANTS.update({
+ 'SOURCE_%sWHL' % name_infix: ('%%(name)s-%%(version)s-%s-none-any.whl' % pyver,
+ 'Generic (non-compiled) %s wheel package' % desc),
+ 'SOURCELOWER_%sWHL' % name_infix: ('%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver,
+ 'Generic (non-compiled) %s wheel package with lowercase name' % desc),
+ })
# TODO derived config templates
# versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) )
@@ -188,10 +287,9 @@
def template_constant_dict(config, ignore=None, toolchain=None):
"""Create a dict for templating the values in the easyconfigs.
- - config is a dict with the structure of EasyConfig._config
+ - config -- Dict with the structure of EasyConfig._config
+ - ignore -- List of template names to ignore
"""
- # TODO find better name
- # ignore
if ignore is None:
ignore = []
# make dict
@@ -202,19 +300,25 @@ def template_constant_dict(config, ignore=None, toolchain=None):
# set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value
template_values['arch'] = platform.uname()[4]
+ # set 'rpath' template based on 'rpath' configuration option, using empty string as fallback
+ template_values['rpath_enabled'] = 'true' if build_option('rpath') else 'false'
+
# set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback
template_values['sysroot'] = build_option('sysroot') or ''
+ # set 'software_commit' template based on 'software_commit' configuration option, default to None
+ template_values['software_commit'] = build_option('software_commit') or ''
+
# step 1: add TEMPLATE_NAMES_EASYCONFIG
for name in TEMPLATE_NAMES_EASYCONFIG:
if name in ignore:
continue
# check if this template name is already handled
- if template_values.get(name[0]) is not None:
+ if template_values.get(name) is not None:
continue
- if name[0].startswith('toolchain_'):
+ if name.startswith('toolchain_'):
tc = config.get('toolchain')
if tc is not None:
template_values['toolchain_name'] = tc.get('name', None)
@@ -222,7 +326,7 @@ def template_constant_dict(config, ignore=None, toolchain=None):
# only go through this once
ignore.extend(['toolchain_name', 'toolchain_version'])
- elif name[0].startswith('version_'):
+ elif name.startswith('version_'):
# parse major and minor version numbers
version = config['version']
if version is not None:
@@ -241,87 +345,85 @@ def template_constant_dict(config, ignore=None, toolchain=None):
# only go through this once
ignore.extend(['version_major', 'version_minor', 'version_major_minor'])
- elif name[0].endswith('letter'):
+ elif name.endswith('letter'):
# parse first letters
- if name[0].startswith('name'):
+ if name.startswith('name'):
softname = config['name']
if softname is not None:
template_values['nameletter'] = softname[0]
- elif name[0] == 'module_name':
+ elif name == 'module_name':
template_values['module_name'] = getattr(config, 'short_mod_name', None)
else:
raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name)
# step 2: define *ver and *shortver templates
- if TEMPLATE_SOFTWARE_VERSIONS:
-
- name_to_prefix = dict((name.lower(), pref) for name, pref in TEMPLATE_SOFTWARE_VERSIONS)
- deps = config.get('dependencies', [])
-
- # also consider build dependencies for *ver and *shortver templates;
- # we need to be a bit careful here, because for iterative installations
- # (when multi_deps is used for example) the builddependencies value may be a list of lists
-
- # first, determine if we have an EasyConfig instance
- # (indirectly by checking for 'iterating' and 'iterate_options' attributes,
- # because we can't import the EasyConfig class here without introducing
- # a cyclic import...);
- # we need to know to determine whether we're iterating over a list of build dependencies
- is_easyconfig = hasattr(config, 'iterating') and hasattr(config, 'iterate_options')
- if is_easyconfig:
- # if we're iterating over different lists of build dependencies,
- # only consider build dependencies when we're actually in iterative mode!
- if 'builddependencies' in config.iterate_options:
- if config.iterating:
- build_deps = config.get('builddependencies')
- else:
- build_deps = None
- else:
+ name_to_prefix = {name.lower(): prefix for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items()}
+ deps = config.get('dependencies', [])
+
+ # also consider build dependencies for *ver and *shortver templates;
+ # we need to be a bit careful here, because for iterative installations
+ # (when multi_deps is used for example) the builddependencies value may be a list of lists
+
+ # first, determine if we have an EasyConfig instance
+ # (indirectly by checking for 'iterating' and 'iterate_options' attributes,
+ # because we can't import the EasyConfig class here without introducing
+ # a cyclic import...);
+ # we need to know to determine whether we're iterating over a list of build dependencies
+ is_easyconfig = hasattr(config, 'iterating') and hasattr(config, 'iterate_options')
+ if is_easyconfig:
+ # if we're iterating over different lists of build dependencies,
+ # only consider build dependencies when we're actually in iterative mode!
+ if 'builddependencies' in config.iterate_options:
+ if config.iterating:
build_deps = config.get('builddependencies')
+ else:
+ build_deps = None
+ else:
+ build_deps = config.get('builddependencies')
+ if build_deps:
+ # Don't use += to avoid changing original list
+ deps = deps + build_deps
+ # include all toolchain deps (e.g. CUDAcore component in fosscuda);
+ # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain!
+ if config._toolchain is not None and config._toolchain.tcdeps:
+ # If we didn't create a new list above do it here
if build_deps:
- # Don't use += to avoid changing original list
- deps = deps + build_deps
- # include all toolchain deps (e.g. CUDAcore component in fosscuda);
- # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain!
- if config._toolchain is not None and config._toolchain.tcdeps:
- # If we didn't create a new list above do it here
- if build_deps:
- deps.extend(config._toolchain.tcdeps)
- else:
- deps = deps + config._toolchain.tcdeps
-
- for dep in deps:
- if isinstance(dep, dict):
- dep_name, dep_version = dep['name'], dep['version']
-
- # take into account dependencies marked as external modules,
- # where name/version may have to be harvested from metadata available for that external module
- if dep.get('external_module', False):
- metadata = dep.get('external_module_metadata', {})
- if dep_name is None:
- # name is a list in metadata, just take first value (if any)
- dep_name = metadata.get('name', [None])[0]
- if dep_version is None:
- # version is a list in metadata, just take first value (if any)
- dep_version = metadata.get('version', [None])[0]
-
- elif isinstance(dep, (list, tuple)):
- dep_name, dep_version = dep[0], dep[1]
+ deps.extend(config._toolchain.tcdeps)
else:
- raise EasyBuildError("Unexpected type for dependency: %s", dep)
-
- if isinstance(dep_name, str) and dep_version:
- pref = name_to_prefix.get(dep_name.lower())
- if pref:
- dep_version = pick_dep_version(dep_version)
- template_values['%sver' % pref] = dep_version
- dep_version_parts = dep_version.split('.')
- template_values['%smajver' % pref] = dep_version_parts[0]
- if len(dep_version_parts) > 1:
- template_values['%sminver' % pref] = dep_version_parts[1]
- template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2])
+ deps = deps + config._toolchain.tcdeps
+
+ for dep in deps:
+ if isinstance(dep, dict):
+ dep_name, dep_version = dep['name'], dep['version']
+
+ # take into account dependencies marked as external modules,
+ # where name/version may have to be harvested from metadata available for that external module
+ if dep.get('external_module', False):
+ metadata = dep.get('external_module_metadata', {})
+ if dep_name is None:
+ # name is a list in metadata, just take first value (if any)
+ dep_name = metadata.get('name', [None])[0]
+ if dep_version is None:
+ # version is a list in metadata, just take first value (if any)
+ dep_version = metadata.get('version', [None])[0]
+
+ elif isinstance(dep, (list, tuple)):
+ dep_name, dep_version = dep[0], dep[1]
+ else:
+ raise EasyBuildError("Unexpected type for dependency: %s", dep)
+
+ if isinstance(dep_name, str) and dep_version:
+ pref = name_to_prefix.get(dep_name.lower())
+ if pref:
+ dep_version = pick_dep_version(dep_version)
+ template_values['%sver' % pref] = dep_version
+ dep_version_parts = dep_version.split('.')
+ template_values['%smajver' % pref] = dep_version_parts[0]
+ if len(dep_version_parts) > 1:
+ template_values['%sminver' % pref] = dep_version_parts[1]
+ template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2])
# step 3: add remaining from config
for name in TEMPLATE_NAMES_CONFIG:
@@ -362,24 +464,24 @@ def template_constant_dict(config, ignore=None, toolchain=None):
# step 6. CUDA compute capabilities
# Use the commandline / easybuild config option if given, else use the value from the EC (as a default)
- cuda_compute_capabilities = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities')
- if cuda_compute_capabilities:
- template_values['cuda_compute_capabilities'] = ','.join(cuda_compute_capabilities)
- template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities)
- template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities)
- template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities)
- int_values = [cc.replace('.', '') for cc in cuda_compute_capabilities]
+ cuda_cc = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities')
+ if cuda_cc:
+ template_values['cuda_compute_capabilities'] = ','.join(cuda_cc)
+ template_values['cuda_cc_space_sep'] = ' '.join(cuda_cc)
+ template_values['cuda_cc_space_sep_no_period'] = ' '.join(cc.replace('.', '') for cc in cuda_cc)
+ template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_cc)
+ template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_cc)
+ int_values = [cc.replace('.', '') for cc in cuda_cc]
template_values['cuda_int_comma_sep'] = ','.join(int_values)
template_values['cuda_int_space_sep'] = ' '.join(int_values)
template_values['cuda_int_semicolon_sep'] = ';'.join(int_values)
- sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities]
+ sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_cc]
template_values['cuda_sm_comma_sep'] = ','.join(sm_values)
template_values['cuda_sm_space_sep'] = ' '.join(sm_values)
unknown_names = []
for key in template_values:
- dynamic_template_names = set(x for (x, _) in TEMPLATE_NAMES_DYNAMIC)
- if not (key in common_template_names or key in dynamic_template_names):
+ if not (key in common_template_names or key in TEMPLATE_NAMES_DYNAMIC):
unknown_names.append(key)
if unknown_names:
raise EasyBuildError("One or more template values found with unknown name: %s", ','.join(unknown_names))
@@ -429,15 +531,15 @@ def template_documentation():
# step 1: add TEMPLATE_NAMES_EASYCONFIG
doc.append('Template names/values derived from easyconfig instance')
- for name in TEMPLATE_NAMES_EASYCONFIG:
- doc.append("%s%%(%s)s: %s" % (indent_l1, name[0], name[1]))
+ for name, cur_doc in TEMPLATE_NAMES_EASYCONFIG.items():
+ doc.append("%s%%(%s)s: %s" % (indent_l1, name, cur_doc))
# step 2: add *ver/*shortver templates for software listed in TEMPLATE_SOFTWARE_VERSIONS
doc.append("Template names/values for (short) software versions")
- for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
- doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, pref, name))
- doc.append("%s%%(%sshortver)s: short version for %s (.)" % (indent_l1, pref, name))
- doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, pref, name))
+ for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items():
+ doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, prefix, name))
+ doc.append("%s%%(%sshortver)s: short version for %s (.)" % (indent_l1, prefix, name))
+ doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, prefix, name))
# step 3: add remaining self._config
doc.append('Template names/values as set in easyconfig')
@@ -453,11 +555,16 @@ def template_documentation():
# step 5. self.template_values can/should be updated from outside easyconfig
# (eg the run_setp code in EasyBlock)
doc.append('Template values set outside EasyBlock runstep')
- for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
- doc.append("%s%%(%s)s: %s" % (indent_l1, name[0], name[1]))
+ for name, cur_doc in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.items():
+ doc.append("%s%%(%s)s: %s" % (indent_l1, name, cur_doc))
doc.append('Template constants that can be used in easyconfigs')
- for cst in TEMPLATE_CONSTANTS:
- doc.append('%s%s: %s (%s)' % (indent_l1, cst[0], cst[2], cst[1]))
+ for name, (value, cur_doc) in TEMPLATE_CONSTANTS.items():
+ doc.append('%s%s: %s (%s)' % (indent_l1, name, cur_doc, value))
return "\n".join(doc)
+
+
+# Add template constants to export list
+globals().update({name: value for name, (value, _) in TEMPLATE_CONSTANTS.items()})
+__all__ = list(TEMPLATE_CONSTANTS.keys())
diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py
index d5c482fecd..9c832bd196 100644
--- a/easybuild/framework/easyconfig/tools.py
+++ b/easybuild/framework/easyconfig/tools.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -54,14 +54,15 @@
from easybuild.framework.easyconfig.easyconfig import process_easyconfig
from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_error, print_msg, print_warning
from easybuild.tools.config import build_option
from easybuild.tools.environment import restore_env
-from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files
+from easybuild.tools.filetools import find_easyconfigs, get_cwd, is_patch_file, locate_files
from easybuild.tools.filetools import read_file, resolve_path, which, write_file
from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO
-from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data
-from easybuild.tools.github import fetch_files_from_pr
+from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit
+from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data
+from easybuild.tools.github import fetch_files_from_commit, fetch_files_from_pr
from easybuild.tools.multidiff import multidiff
from easybuild.tools.toolchain.toolchain import is_system_toolchain
from easybuild.tools.toolchain.utilities import search_toolchain
@@ -217,10 +218,12 @@ def mk_node_name(spec):
if dep in spec['ec'].build_dependencies:
dgr.add_edge_attributes((spec['module'], dep), attrs=edge_attrs)
- _dep_graph_dump(dgr, filename)
-
- if not build_option('silent'):
- print("Wrote dependency graph for %d easyconfigs to %s" % (len(specs), filename))
+ what = "dependency graph for %d easyconfigs to %s" % (len(specs), filename)
+ silent = build_option('silent')
+ if _dep_graph_dump(dgr, filename):
+ print_msg("Wrote " + what, silent=silent)
+ else:
+ print_error("Failed writing " + what, silent=silent)
@only_if_module_is_available('pygraph.readwrite.dot', pkgname='python-graph-dot')
@@ -230,9 +233,15 @@ def _dep_graph_dump(dgr, filename):
dottxt = dot.write(dgr)
if os.path.splitext(filename)[-1] == '.dot':
# create .dot file
- write_file(filename, dottxt)
+ try:
+ write_file(filename, dottxt)
+ except EasyBuildError as e:
+ print(str(e))
+ return False
+ else:
+ return True
else:
- _dep_graph_gv(dottxt, filename)
+ return _dep_graph_gv(dottxt, filename)
@only_if_module_is_available('gv', pkgname='graphviz-python')
@@ -240,8 +249,8 @@ def _dep_graph_gv(dottxt, filename):
"""Render dependency graph to file using graphviz."""
# try and render graph in specified file format
gvv = gv.readstring(dottxt)
- gv.layout(gvv, 'dot')
- gv.render(gvv, os.path.splitext(filename)[-1], filename)
+ if gv.layout(gvv, 'dot') is not False:
+ return gv.render(gvv, os.path.splitext(filename)[-1], filename)
def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None):
@@ -309,7 +318,7 @@ def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None):
return paths
-def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=None):
+def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, from_commit=None, review_pr=None):
"""Obtain alternative paths for easyconfig files."""
# paths where tweaked easyconfigs will be placed, easyconfigs listed on the command line take priority and will be
@@ -320,18 +329,20 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=Non
tweaked_ecs_paths = (os.path.join(tmpdir, 'tweaked_easyconfigs'),
os.path.join(tmpdir, 'tweaked_dep_easyconfigs'))
- # paths where files touched in PRs will be downloaded to,
- # which are picked up via 'pr_paths' build option in fetch_files_from_pr
- pr_paths = []
+ # paths where files touched in commit/PRs will be downloaded to,
+ # which are picked up via 'extra_ec_paths' build option in fetch_files_from_pr
+ extra_ec_paths = []
if from_prs:
- pr_paths = from_prs[:]
- if review_pr and review_pr not in pr_paths:
- pr_paths.append(review_pr)
+ extra_ec_paths = from_prs[:]
+ if review_pr and review_pr not in extra_ec_paths:
+ extra_ec_paths.append(review_pr)
+ if extra_ec_paths:
+ extra_ec_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in extra_ec_paths]
- if pr_paths:
- pr_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in pr_paths]
+ if from_commit:
+ extra_ec_paths.append(os.path.join(tmpdir, 'files_commit_' + from_commit))
- return tweaked_ecs_paths, pr_paths
+ return tweaked_ecs_paths, extra_ec_paths
def det_easyconfig_paths(orig_paths):
@@ -345,27 +356,31 @@ def det_easyconfig_paths(orig_paths):
except ValueError:
raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.")
+ from_commit = build_option('from_commit')
robot_path = build_option('robot_path')
# list of specified easyconfig files
ec_files = orig_paths[:]
+ commit_files, pr_files = [], []
if from_prs:
- pr_files = []
for pr in from_prs:
- # path to where easyconfig files should be downloaded is determined via 'pr_paths' build option,
- # which corresponds to the list of PR paths returned by alt_easyconfig_paths
+ # path to where easyconfig files should be downloaded is determined
+ # via 'extra_ec_paths' build options,
+ # which corresponds to the list of commit/PR paths returned by alt_easyconfig_paths
pr_files.extend(fetch_easyconfigs_from_pr(pr))
-
- if ec_files:
- # replace paths for specified easyconfigs that are touched in PR
- for i, ec_file in enumerate(ec_files):
- for pr_file in pr_files:
- if ec_file == os.path.basename(pr_file):
- ec_files[i] = pr_file
- else:
- # if no easyconfigs are specified, use all the ones touched in the PR
- ec_files = [path for path in pr_files if path.endswith('.eb')]
+ elif from_commit:
+ commit_files = fetch_easyconfigs_from_commit(from_commit, files=ec_files)
+
+ if ec_files:
+ # replace paths for specified easyconfigs that are touched in commit/PRs
+ for i, ec_file in enumerate(ec_files):
+ for file in commit_files + pr_files:
+ if ec_file == os.path.basename(file):
+ ec_files[i] = file
+ else:
+ # if no easyconfigs are specified, use all the ones touched in the commit/PRs
+ ec_files = [path for path in commit_files + pr_files if path.endswith('.eb')]
filter_ecs = build_option('filter_ecs')
if filter_ecs:
@@ -394,7 +409,7 @@ def parse_easyconfigs(paths, validate=True):
# keep track of whether any files were generated
generated_ecs |= generated
if not os.path.exists(path):
- raise EasyBuildError("Can't find path %s", path)
+ raise EasyBuildError("Can't find path %s", path, exit_code=EasyBuildExit.MISSING_EASYCONFIG)
try:
ec_files = find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs'))
for ec_file in ec_files:
@@ -777,7 +792,7 @@ def avail_easyblocks():
return easyblocks
-def det_copy_ec_specs(orig_paths, from_pr):
+def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None):
"""Determine list of paths + target directory for --copy-ec."""
if from_pr is not None and not isinstance(from_pr, list):
@@ -785,14 +800,13 @@ def det_copy_ec_specs(orig_paths, from_pr):
target_path, paths = None, []
- # if only one argument is specified, use current directory as target directory
if len(orig_paths) == 1:
- target_path = os.getcwd()
+ # if only one argument is specified, use current directory as target directory
+ target_path = get_cwd()
paths = orig_paths[:]
-
- # if multiple arguments are specified, assume that last argument is target location,
- # and remove that from list of paths to copy
elif orig_paths:
+ # if multiple arguments are specified, assume that last argument is target location,
+ # and remove that from list of paths to copy
target_path = orig_paths[-1]
paths = orig_paths[:-1]
@@ -810,7 +824,7 @@ def det_copy_ec_specs(orig_paths, from_pr):
pr_paths.extend(fetch_files_from_pr(pr=pr, path=tmpdir))
# assume that files need to be copied to current working directory for now
- target_path = os.getcwd()
+ target_path = get_cwd()
if orig_paths:
last_path = orig_paths[-1]
@@ -841,4 +855,41 @@ def det_copy_ec_specs(orig_paths, from_pr):
elif pr_matches:
raise EasyBuildError("Found multiple paths for %s in PR: %s", filename, pr_matches)
+ # consider --from-commit (only if --from-pr was not used)
+ elif from_commit:
+ tmpdir = os.path.join(tempfile.gettempdir(), 'fetch_files_from_commit_%s' % from_commit)
+ commit_paths = fetch_files_from_commit(from_commit, path=tmpdir)
+
+ # assume that files need to be copied to current working directory for now
+ target_path = get_cwd()
+
+ if orig_paths:
+ last_path = orig_paths[-1]
+
+ # check files touched by commit and see if the target directory for --copy-ec
+ # corresponds to the name of one of these files;
+ # if so we should copy the specified file(s) to the current working directory,
+ # since interpreting the last argument as target location is very unlikely to be correct in this case
+ commit_filenames = [os.path.basename(p) for p in commit_paths]
+ if last_path in commit_filenames:
+ paths = orig_paths[:]
+ else:
+ target_path = last_path
+ # exclude last argument that is used as target location
+ paths = orig_paths[:-1]
+
+ # if list of files to copy is empty at this point,
+ # we simply copy *all* files touched by the PR
+ if not paths:
+ paths = commit_paths
+
+ # replace path for files touched by commit (no need to worry about others)
+ for idx, path in enumerate(paths):
+ filename = os.path.basename(path)
+ commit_matches = [x for x in commit_paths if os.path.basename(x) == filename]
+ if len(commit_matches) == 1:
+ paths[idx] = commit_matches[0]
+ elif commit_matches:
+ raise EasyBuildError("Found multiple paths for %s in commit: %s", filename, commit_matches)
+
return paths, target_path
diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py
index c27c8f8728..2a88e56d6a 100644
--- a/easybuild/framework/easyconfig/tweak.py
+++ b/easybuild/framework/easyconfig/tweak.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -1236,7 +1236,7 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin
(highest_version_ignoring_versionsuffix is not None and highest_version is not None and
LooseVersion(highest_version_ignoring_versionsuffix) > LooseVersion(highest_version))
- exclude_alternate_versionsuffixes = False
+ exclude_alternative_versionsuffixes = False
if ignored_versionsuffix_greater:
if ignore_versionsuffixes:
highest_version = highest_version_ignoring_versionsuffix
@@ -1247,11 +1247,11 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin
dep['name'], versionsuffix, [d['path'] for d in potential_version_mappings if
d['version'] == highest_version_ignoring_versionsuffix])
# exclude candidates with a different versionsuffix
- exclude_alternate_versionsuffixes = True
+ exclude_alternative_versionsuffixes = True
else:
# If the other version suffixes are not greater, then just ignore them
- exclude_alternate_versionsuffixes = True
- if exclude_alternate_versionsuffixes:
+ exclude_alternative_versionsuffixes = True
+ if exclude_alternative_versionsuffixes:
potential_version_mappings = [d for d in potential_version_mappings if d['versionsuffix'] == versionsuffix]
if highest_versions_only and highest_version is not None:
diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py
index fa9829e312..f902716dfa 100644
--- a/easybuild/framework/easyconfig/types.py
+++ b/easybuild/framework/easyconfig/types.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -31,7 +31,6 @@
* Caroline De Brouwer (Ghent University)
* Kenneth Hoste (Ghent University)
"""
-from distutils.util import strtobool
from easybuild.base import fancylogger
from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
@@ -280,7 +279,14 @@ def to_toolchain_dict(spec):
res = {'name': spec[0].strip(), 'version': spec[1].strip()}
# 3-element list
elif len(spec) == 3:
- res = {'name': spec[0].strip(), 'version': spec[1].strip(), 'hidden': strtobool(spec[2].strip())}
+ hidden = spec[2].strip().lower()
+ if hidden in {'yes', 'true', 't', 'y', '1', 'on'}:
+ hidden = True
+ elif hidden in {'no', 'false', 'f', 'n', '0', 'off'}:
+ hidden = False
+ else:
+ raise EasyBuildError("Invalid truth value %s", hidden)
+ res = {'name': spec[0].strip(), 'version': spec[1].strip(), 'hidden': hidden}
else:
raise EasyBuildError("Can not convert list %s to toolchain dict. Expected 2 or 3 elements", spec)
@@ -510,7 +516,7 @@ def to_checksums(checksums):
for checksum in checksums:
# each list entry can be:
# * None (indicates no checksum)
- # * a string (MD5 or SHA256 checksum)
+ # * a string (SHA256 checksum)
# * a tuple with 2 elements: checksum type + checksum value
# * a list of checksums (i.e. multiple checksums for a single file)
# * a dict (filename to checksum mapping)
diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py
index 9ab411c918..0c5909cdc4 100644
--- a/easybuild/framework/easystack.py
+++ b/easybuild/framework/easystack.py
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Ghent University
+# Copyright 2020-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -188,9 +188,6 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa
@only_if_module_is_available('yaml', pkgname='PyYAML')
def parse_easystack(filepath):
"""Parses through easystack file, returns what EC are to be installed together with their options."""
- log_msg = "Support for easybuild-ing from multiple easyconfigs based on "
- log_msg += "information obtained from provided file (easystack) with build specifications."
- _log.experimental(log_msg)
_log.info("Building from easystack: '%s'" % filepath)
# class instance which contains all info about planned build
diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py
index 9f099eb74c..0005f24428 100644
--- a/easybuild/framework/extension.py
+++ b/easybuild/framework/extension.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,7 +40,7 @@
from easybuild.framework.easyconfig.easyconfig import resolve_template
from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
-from easybuild.tools.build_log import EasyBuildError, raise_nosupport
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, raise_nosupport
from easybuild.tools.filetools import change_dir
from easybuild.tools.run import run_shell_cmd
@@ -116,7 +116,7 @@ def __init__(self, mself, ext, extra_params=None):
# Add install/builddir templates with values from master.
for key in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
- self.cfg.template_values[key[0]] = str(getattr(self.master, key[0], None))
+ self.cfg.template_values[key] = str(getattr(self.master, key, None))
# We can't inherit the 'start_dir' value from the parent (which will be set, and will most likely be wrong).
# It should be specified for the extension specifically, or be empty (so it is auto-derived).
@@ -167,29 +167,99 @@ def version(self):
return self.ext.get('version', None)
def prerun(self):
+ """
+ [DEPRECATED][6.0] Stuff to do before installing a extension.
+ """
+ # Deprecation warning triggered by Extension.install_extension_substep()
+ self.pre_install_extension()
+
+ def pre_install_extension(self):
"""
Stuff to do before installing a extension.
"""
pass
def run(self, *args, **kwargs):
+ """
+ [DEPRECATED][6.0] Actual installation of an extension.
+ """
+ # Deprecation warning triggered by Extension.install_extension_substep()
+ self.install_extension(*args, **kwargs)
+
+ def install_extension(self, *args, **kwargs):
"""
Actual installation of an extension.
"""
pass
def run_async(self, *args, **kwargs):
+ """
+ [DEPRECATED][6.0] Asynchronous installation of an extension.
+ """
+ # Deprecation warning triggered by Extension.install_extension_substep()
+ self.install_extension_async(*args, **kwargs)
+
+ def install_extension_async(self, *args, **kwargs):
"""
Asynchronous installation of an extension.
"""
raise NotImplementedError
def postrun(self):
+ """
+ [DEPRECATED][6.0] Stuff to do after installing a extension.
+ """
+ # Deprecation warning triggered by Extension.install_extension_substep()
+ self.post_install_extension()
+
+ def post_install_extension(self):
"""
Stuff to do after installing a extension.
"""
self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', []))
+ def install_extension_substep(self, substep, *args, **kwargs):
+ """
+ Carry out extension installation substep allowing use of deprecated
+ methods on those extensions using an older EasyBlock
+ """
+ substeps_mapping = {
+ 'pre_install_extension': 'prerun',
+ 'install_extension': 'run',
+ 'install_extension_async': 'run_async',
+ 'post_install_extension': 'postrun',
+ }
+
+ deprecated_substep = substeps_mapping.get(substep)
+ if deprecated_substep is None:
+ raise EasyBuildError("Unknown extension installation substep: %s", substep)
+
+ try:
+ substep_method = getattr(self, deprecated_substep)
+ except AttributeError:
+ log_msg = f"EasyBlock does not implement deprecated method '{deprecated_substep}' "
+ log_msg += f"for installation substep {substep}"
+ self.log.debug(log_msg)
+ substep_method = getattr(self, substep)
+ else:
+ # Qualified method name contains class defining the method (PEP 3155)
+ substep_method_name = substep_method.__qualname__
+ self.log.debug(f"Found deprecated method in EasyBlock: {substep_method_name}")
+
+ base_method_name = f"Extension.{deprecated_substep}"
+ if substep_method_name == base_method_name:
+ # No custom method in child Easyblock, deprecated method is defined by base Extension class
+ # Switch to non-deprecated substep method
+ substep_method = getattr(self, substep)
+ else:
+ # Custom deprecated method used by child Easyblock
+ self.log.deprecated(
+ f"{substep_method_name}() is deprecated, use {substep}() instead.",
+ "6.0",
+ )
+
+ return substep_method(*args, **kwargs)
+
@property
def required_deps(self):
"""Return list of required dependencies for this extension."""
@@ -232,7 +302,7 @@ def sanity_check_step(self):
cmd, stdin = resolve_exts_filter_template(exts_filter, self)
cmd_res = run_shell_cmd(cmd, fail_on_error=False, stdin=stdin)
- if cmd_res.exit_code:
+ if cmd_res.exit_code != EasyBuildExit.SUCCESS:
if stdin:
fail_msg = 'command "%s" (stdin: "%s") failed' % (cmd, stdin)
else:
diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py
index 6ad116b20b..dca2d587cc 100644
--- a/easybuild/framework/extensioneasyblock.py
+++ b/easybuild/framework/extensioneasyblock.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of the University of Ghent (http://ugent.be/hpc).
@@ -126,18 +126,23 @@ def _set_start_dir(self):
elif ext_start_dir is None:
# This may be on purpose, e.g. for Python WHL files which do not get extracted
self.log.debug("Start dir is not set.")
- else:
+ elif self.start_dir:
# non-existing start dir means wrong input from user
- warn_msg = "Provided start dir (%s) for extension %s does not exist: %s" % (self.start_dir, self.name,
- ext_start_dir)
+ raise EasyBuildError("Provided start dir (%s) for extension %s does not exist: %s",
+ self.start_dir, self.name, ext_start_dir)
+ else:
+ warn_msg = 'Failed to determine start dir for extension %s: %s' % (self.name, ext_start_dir)
self.log.warning(warn_msg)
print_warning(warn_msg, silent=build_option('silent'))
- def run(self, unpack_src=False):
+ def install_extension(self, unpack_src=False):
"""Common operations for extensions: unpacking sources, patching, ..."""
# unpack file if desired
- if unpack_src:
+ if self.options.get('nosource', False):
+ # If no source wanted use the start_dir from the main EC
+ self.ext_dir = self.master.start_dir
+ elif unpack_src:
targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name))
self.ext_dir = extract_file(self.src, targetdir, extra_options=self.unpack_options,
change_into_dir=False, cmd=self.src_extract_cmd)
@@ -146,10 +151,9 @@ def run(self, unpack_src=False):
# because start_dir value is usually a relative path (if it is set)
change_dir(self.ext_dir)
- self._set_start_dir()
+ self._set_start_dir()
+ if self.start_dir:
change_dir(self.start_dir)
- else:
- self._set_start_dir()
# patch if needed
EasyBlock.patch_step(self, beginpath=self.ext_dir)
diff --git a/easybuild/main.py b/easybuild/main.py
index 9272c7abef..83836d22c3 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -82,6 +82,8 @@
from easybuild.tools.repository.repository import init_repository
from easybuild.tools.systemtools import check_easybuild_deps
from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
+from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION, UNKNOWN_EASYBLOCKS_VERSION
+from easybuild.tools.version import different_major_versions
_log = None
@@ -131,10 +133,10 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
ec_res = {}
try:
- (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env)
+ (ec_res['success'], app_log, err_msg, err_code) = build_and_install_one(ec, init_env)
ec_res['log_file'] = app_log
if not ec_res['success']:
- ec_res['err'] = EasyBuildError(err)
+ ec_res['err'] = EasyBuildError(err_msg, exit_code=err_code)
except Exception as err:
# purposely catch all exceptions
ec_res['success'] = False
@@ -172,7 +174,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
if not isinstance(ec_res['err'], EasyBuildError):
raise ec_res['err']
else:
- raise EasyBuildError(test_msg)
+ raise EasyBuildError(test_msg, exit_code=err_code)
res.append((ec, ec_res))
@@ -328,7 +330,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
if options.copy_ec:
# figure out list of files to copy + target location (taking into account --from-pr)
- eb_args, target_path = det_copy_ec_specs(eb_args, from_pr_list)
+ eb_args, target_path = det_copy_ec_specs(eb_args, from_pr=from_pr_list, from_commit=options.from_commit)
categorized_paths = categorize_files_by_type(eb_args)
@@ -439,9 +441,9 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules
keep_available_modules = any((
- forced, dry_run_mode, options.extended_dry_run, any_pr_option_set, options.copy_ec, options.inject_checksums,
- options.sanity_check_only, options.inject_checksums_to_json)
- )
+ forced, dry_run_mode, any_pr_option_set, options.copy_ec, options.dump_env_script, options.extended_dry_run,
+ options.inject_checksums, options.inject_checksums_to_json, options.sanity_check_only
+ ))
# skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
if not keep_available_modules:
@@ -507,7 +509,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
# dry_run: print all easyconfigs and dependencies, and whether they are already built
elif dry_run_mode:
if options.missing_modules:
- txt = missing_deps(easyconfigs, modtool)
+ txt = missing_deps(easyconfigs, modtool, terse=options.terse)
else:
txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
print_msg(txt, log=_log, silent=testing, prefix=False)
@@ -614,6 +616,15 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr
(build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate,
from_pr_list, tweaked_ecs_paths) = cfg_settings
+ # compare running Framework and EasyBlocks versions
+ if EASYBLOCKS_VERSION == UNKNOWN_EASYBLOCKS_VERSION:
+ # most likely reason is running framework unit tests with no easyblocks installation
+ # so log a warning, to avoid test related issues
+ _log.warning("Unable to determine EasyBlocks version, so we'll assume it is not different from Framework")
+ elif different_major_versions(FRAMEWORK_VERSION, EASYBLOCKS_VERSION):
+ raise EasyBuildError("Framework (%s) and EasyBlock (%s) major versions are different." % (FRAMEWORK_VERSION,
+ EASYBLOCKS_VERSION))
+
# load hook implementations (if any)
hooks = load_hooks(options.hooks)
@@ -693,7 +704,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr
options.list_prs,
options.merge_pr,
options.review_pr,
- options.terse,
+ # --missing-modules is processed by process_eb_args,
+ # so we can't exit just yet here if it's used in combination with --terse
+ options.terse and not options.missing_modules,
search_query,
]
if any(early_stop_options):
@@ -710,9 +723,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr
if options.ignore_test_failure:
raise EasyBuildError("Found both ignore-test-failure and skip-test-step enabled. "
"Please use only one of them.")
- else:
- print_warning("Will not run the test step as requested via skip-test-step. "
- "Consider using ignore-test-failure instead and verify the results afterwards")
+ print_warning("Will not run the test step as requested via skip-test-step. "
+ "Consider using ignore-test-failure instead and verify the results afterwards")
+ if options.skip_sanity_check and options.sanity_check_only:
+ raise EasyBuildError("Found both skip-sanity-check and sanity-check-only enabled. "
+ "Please use only one of them.")
# if EasyStack file is provided, parse it, and loop over the items in the EasyStack file
if options.easystack:
@@ -730,7 +745,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
stop_logging(logfile, logtostdout=options.logtostdout)
if do_cleanup:
- cleanup(logfile, eb_tmpdir, testing, silent=False)
+ cleanup(logfile, eb_tmpdir, testing, silent=options.terse)
def prepare_main(args=None, logfile=None, testing=None):
@@ -745,8 +760,7 @@ def prepare_main(args=None, logfile=None, testing=None):
# if $CDPATH is set, unset it, it'll only cause trouble...
# see https://github.com/easybuilders/easybuild-framework/issues/2944
- if 'CDPATH' in os.environ:
- del os.environ['CDPATH']
+ os.environ.pop('CDPATH', None)
# When EB is run via `exec` the special bash variable $_ is not set
# So emulate this here to allow (module) scripts depending on that to work
@@ -761,13 +775,19 @@ def prepare_main(args=None, logfile=None, testing=None):
def main_with_hooks(args=None):
- init_session_state, eb_go, cfg_settings = prepare_main(args=args)
+ # take into account that EasyBuildError may be raised when parsing the EasyBuild configuration
+ try:
+ init_session_state, eb_go, cfg_settings = prepare_main(args=args)
+ except EasyBuildError as err:
+ print_error(err.msg, exit_code=err.exit_code)
+
hooks = load_hooks(eb_go.options.hooks)
+
try:
main(args=args, prepared_cfg_data=(init_session_state, eb_go, cfg_settings))
except EasyBuildError as err:
run_hook(FAIL, hooks, args=[err])
- print_error(err.msg, exit_on_error=True, exit_code=1)
+ print_error(err.msg, exit_on_error=True, exit_code=err.exit_code)
except KeyboardInterrupt as err:
run_hook(CANCEL, hooks, args=[err])
print_error("Cancelled by user: %s" % err)
diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py
index 5cb9eac8f5..b166191c12 100755
--- a/easybuild/scripts/clean_gists.py
+++ b/easybuild/scripts/clean_gists.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
##
-# Copyright 2014 Ward Poelmans
+# Copyright 2014-2024 Ward Poelmans
#
# https://github.com/easybuilders/easybuild
#
diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py
index 2d59d77bc0..ec37b1f3dd 100755
--- a/easybuild/scripts/findPythonDeps.py
+++ b/easybuild/scripts/findPythonDeps.py
@@ -1,5 +1,21 @@
#!/usr/bin/env python
+"""
+Find Python dependencies for a given Python package after loading dependencies specified in an EasyConfig.
+This is intended for writing or updating PythonBundle EasyConfigs:
+ 1. Create a EasyConfig with at least 'Python' as a dependency.
+ When updating to a new toolchain it is a good idea to reduce the dependencies to a minimum
+ as e.g. the new "Python" module might have different packages included.
+ 2. Run this script
+ 3. For each dependency found by this script search existing EasyConfigs for ones providing that Python package.
+ E.g many are contained in Python-bundle-PyPI. Some can be updated from an earlier toolchain.
+ 4. Add those EasyConfigs as dependencies to your new EasyConfig.
+ 5. Rerun this script so it takes the newly provided packages into account.
+ You can do steps 3-5 iteratively adding EasyConfig-dependencies one-by-one.
+ 6. Finally you copy the packages found by this script as "exts_list" into the new EasyConfig.
+ You usually want the list printed as "in install order", the format is already suitable to be copied as-is.
+"""
+
import argparse
import json
import os
@@ -8,12 +24,13 @@
import subprocess
import sys
import tempfile
+import textwrap
from contextlib import contextmanager
from pprint import pprint
try:
import pkg_resources
except ImportError as e:
- print('pkg_resources could not be imported: %s\nYou might need to install setuptools!' % e)
+ print(f'pkg_resources could not be imported: {e}\nYou might need to install setuptools!')
sys.exit(1)
try:
@@ -22,6 +39,7 @@
_canonicalize_regex = re.compile(r"[-_.]+")
def canonicalize_name(name):
+ """Fallback if the import doesn't work with same behavior."""
return _canonicalize_regex.sub("-", name).lower()
@@ -36,16 +54,16 @@ def temporary_directory(*args, **kwargs):
def extract_pkg_name(package_spec):
- return re.split('<|>|=|~', args.package, 1)[0]
+ """Get the package name from a specification such as 'package>=3.42'"""
+ return re.split('<|>|=|~', package_spec, 1)[0]
-def can_run(cmd, argument):
+def can_run(cmd, *arguments):
"""Check if the given cmd and argument can be run successfully"""
- with open(os.devnull, 'w') as FNULL:
- try:
- return subprocess.call([cmd, argument], stdout=FNULL, stderr=subprocess.STDOUT) == 0
- except (subprocess.CalledProcessError, OSError):
- return False
+ try:
+ return subprocess.call([cmd, *arguments], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) == 0
+ except (subprocess.CalledProcessError, OSError):
+ return False
def run_shell_cmd(arguments, action_desc, capture_stderr=True, **kwargs):
@@ -59,13 +77,13 @@ def run_shell_cmd(arguments, action_desc, capture_stderr=True, **kwargs):
if p.returncode != 0:
if err:
err = "\nSTDERR:\n" + err
- raise RuntimeError('Failed to %s: %s%s' % (action_desc, out, err))
+ raise RuntimeError(f'Failed to {action_desc}: {out}{err}')
return out
def run_in_venv(cmd, venv_path, action_desc):
"""Run the given command in the virtualenv at the given path"""
- cmd = 'source %s/bin/activate && %s' % (venv_path, cmd)
+ cmd = f'source {venv_path}/bin/activate && {cmd}'
return run_shell_cmd(cmd, action_desc, shell=True, executable='/bin/bash')
@@ -78,43 +96,59 @@ def get_dep_tree(package_spec, verbose):
venv_dir = os.path.join(tmp_dir, 'venv')
if verbose:
print('Creating virtualenv at ' + venv_dir)
- run_shell_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv')
+ run_shell_cmd(
+ [sys.executable, '-m', 'venv', '--system-site-packages', venv_dir], action_desc='create virtualenv'
+ )
if verbose:
print('Updating pip in virtualenv')
run_in_venv('pip install --upgrade pip', venv_dir, action_desc='update pip')
if verbose:
- print('Installing %s into virtualenv' % package_spec)
- out = run_in_venv('pip install "%s"' % package_spec, venv_dir, action_desc='install ' + package_spec)
- print('%s installed: %s' % (package_spec, out))
+ print(f'Installing {package_spec} into virtualenv')
+ out = run_in_venv(f'pip install "{package_spec}"', venv_dir, action_desc='install ' + package_spec)
+ print(f'{package_spec} installed: {out}')
# install pipdeptree, figure out dependency tree for installed package
run_in_venv('pip install pipdeptree', venv_dir, action_desc='install pipdeptree')
- dep_tree = run_in_venv('pipdeptree -j -p "%s"' % package_name,
+ dep_tree = run_in_venv(f'pipdeptree -j -p "{package_name}"',
venv_dir, action_desc='collect dependencies')
return json.loads(dep_tree)
def find_deps(pkgs, dep_tree):
"""Recursively resolve dependencies of the given package(s) and return them"""
+ MAX_PACKAGES = 1000
res = []
- for orig_pkg in pkgs:
- pkg = canonicalize_name(orig_pkg)
- matching_entries = [entry for entry in dep_tree
- if pkg in (entry['package']['package_name'], entry['package']['key'])]
- if not matching_entries:
+ next_pkgs = set(pkgs)
+ # Don't check any package multiple times to avoid infinite recursion
+ seen_pkgs = set()
+ count = 0
+ while next_pkgs:
+ cur_pkgs = next_pkgs - seen_pkgs
+ seen_pkgs.update(cur_pkgs)
+ next_pkgs = set()
+ for orig_pkg in cur_pkgs:
+ count += 1
+ if count > MAX_PACKAGES:
+ raise RuntimeError(f"Aborting after checking {MAX_PACKAGES} packages. Possibly cycle detected!")
+ pkg = canonicalize_name(orig_pkg)
matching_entries = [entry for entry in dep_tree
- if orig_pkg in (entry['package']['package_name'], entry['package']['key'])]
- if not matching_entries:
- raise RuntimeError("Found no installed package for '%s' in %s" % (pkg, dep_tree))
- if len(matching_entries) > 1:
- raise RuntimeError("Found multiple installed packages for '%s' in %s" % (pkg, dep_tree))
- entry = matching_entries[0]
- res.append((entry['package']['package_name'], entry['package']['installed_version']))
- deps = (dep['package_name'] for dep in entry['dependencies'])
- res.extend(find_deps(deps, dep_tree))
+ if pkg in (entry['package']['package_name'], entry['package']['key'])]
+ if not matching_entries:
+ matching_entries = [entry for entry in dep_tree
+ if orig_pkg in (entry['package']['package_name'], entry['package']['key'])]
+ if not matching_entries:
+ raise RuntimeError(f"Found no installed package for '{pkg}' in {dep_tree}")
+ if len(matching_entries) > 1:
+ raise RuntimeError(f"Found multiple installed packages for '{pkg}' in {dep_tree}")
+ entry = matching_entries[0]
+ res.append(entry['package'])
+ # Add dependencies to list of packages to check next
+ # Could call this function recursively but that might exceed the max recursion depth
+ next_pkgs.update(dep['package_name'] for dep in entry['dependencies'])
return res
def print_deps(package, verbose):
+ """Print dependencies of the given package that are not installed yet in a format usable as 'exts_list'"""
if verbose:
print('Getting dep tree of ' + package)
dep_tree = get_dep_tree(package, verbose)
@@ -131,13 +165,18 @@ def print_deps(package, verbose):
res = []
handled = set()
for dep in reversed(deps):
- if dep not in handled:
- handled.add(dep)
- if dep[0] in installed_modules:
+ # Tuple as we need it for exts_list
+ dep_entry = (dep['package_name'], dep['installed_version'])
+ if dep_entry not in handled:
+ handled.add(dep_entry)
+ # Need to check for key and package_name as naming is not consistent. E.g.:
+ # "PyQt5-sip": 'key': 'pyqt5-sip', 'package_name': 'PyQt5-sip'
+ # "jupyter-core": 'key': 'jupyter-core', 'package_name': 'jupyter_core'
+ if dep['key'] in installed_modules or dep['package_name'] in installed_modules:
if verbose:
- print("Skipping installed module '%s'" % dep[0])
+ print(f"Skipping installed module '{dep['package_name']}'")
else:
- res.append(dep)
+ res.append(dep_entry)
print("List of dependencies in (likely) install order:")
pprint(res, indent=4)
@@ -145,68 +184,73 @@ def print_deps(package, verbose):
pprint(sorted(res), indent=4)
-examples = [
- 'Example usage with EasyBuild (after installing dependency modules):',
- '\t' + sys.argv[0] + ' --ec TensorFlow-2.3.4.eb tensorflow==2.3.4',
- 'Which is the same as:',
- '\t' + ' && '.join(['eb TensorFlow-2.3.4.eb --dump-env',
- 'source TensorFlow-2.3.4.env',
- sys.argv[0] + ' tensorflow==2.3.4',
- ]),
-]
-parser = argparse.ArgumentParser(
- description='Find dependencies of Python packages by installing it in a temporary virtualenv. ',
- epilog='\n'.join(examples),
- formatter_class=argparse.RawDescriptionHelpFormatter
-)
-parser.add_argument('package', metavar='python-pkg-spec',
- help='Python package spec, e.g. tensorflow==2.3.4')
-parser.add_argument('--ec', metavar='easyconfig', help='EasyConfig to use as the build environment. '
- 'You need to have dependency modules installed already!')
-parser.add_argument('--verbose', help='Verbose output', action='store_true')
-args = parser.parse_args()
-
-if args.ec:
- if not can_run('eb', '--version'):
- print('EasyBuild not found or executable. Make sure it is in your $PATH when using --ec!')
- sys.exit(1)
- if args.verbose:
- print('Checking with EasyBuild for missing dependencies')
- missing_dep_out = run_shell_cmd(['eb', args.ec, '--missing'],
- capture_stderr=False,
- action_desc='Get missing dependencies')
- excluded_dep = '(%s)' % os.path.basename(args.ec)
- missing_deps = [dep for dep in missing_dep_out.split('\n')
- if dep.startswith('*') and excluded_dep not in dep
- ]
- if missing_deps:
- print('You need to install all modules on which %s depends first!' % args.ec)
- print('\n\t'.join(['Missing:'] + missing_deps))
- sys.exit(1)
-
- # If the --ec argument is a (relative) existing path make it absolute so we can find it after the chdir
- ec_arg = os.path.abspath(args.ec) if os.path.exists(args.ec) else args.ec
- with temporary_directory() as tmp_dir:
- old_dir = os.getcwd()
- os.chdir(tmp_dir)
- if args.verbose:
- print('Running EasyBuild to get build environment')
- run_shell_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment')
- os.chdir(old_dir)
+def main():
+ """Entrypoint of the script"""
+ examples = textwrap.dedent(f"""
+ Example usage with EasyBuild (after installing dependency modules):
+ {sys.argv[0]} --ec TensorFlow-2.3.4.eb tensorflow==2.3.4
+ Which is the same as:
+ eb TensorFlow-2.3.4.eb --dump-env && source TensorFlow-2.3.4.env && {sys.argv[0]} tensorflow==2.3.4
+ Using the '--ec' parameter is recommended as the latter requires manually updating the .env file
+ after each change to the EasyConfig.
+ """)
+ parser = argparse.ArgumentParser(
+ description='Find dependencies of Python packages by installing it in a temporary virtualenv. ',
+ epilog='\n'.join(examples),
+ formatter_class=argparse.RawDescriptionHelpFormatter
+ )
+ parser.add_argument('package', metavar='python-pkg-spec',
+ help='Python package spec, e.g. tensorflow==2.3.4')
+ parser.add_argument('--ec', metavar='easyconfig', help='EasyConfig to use as the build environment. '
+ 'You need to have dependency modules installed already!')
+ parser.add_argument('--verbose', help='Verbose output', action='store_true')
+ args = parser.parse_args()
- cmd = "source %s/*.env && python %s '%s'" % (tmp_dir, sys.argv[0], args.package)
+ if args.ec:
+ if not can_run('eb', '--version'):
+ print('EasyBuild not found or executable. Make sure it is in your $PATH when using --ec!')
+ sys.exit(1)
if args.verbose:
- cmd += ' --verbose'
- print('Restarting script in new build environment')
-
- out = run_shell_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash')
- print(out)
-else:
- if not can_run('virtualenv', '--version'):
- print('Virtualenv not found or executable. ' +
- 'Make sure it is installed (e.g. in the currently loaded Python module)!')
- sys.exit(1)
- if 'PIP_PREFIX' in os.environ:
- print("$PIP_PREFIX is set. Unsetting it as it doesn't work well with virtualenv.")
- del os.environ['PIP_PREFIX']
- print_deps(args.package, args.verbose)
+ print('Checking with EasyBuild for missing dependencies')
+ missing_dep_out = run_shell_cmd(['eb', args.ec, '--missing'],
+ capture_stderr=False,
+ action_desc='Get missing dependencies')
+ excluded_dep = f'({os.path.basename(args.ec)})'
+ missing_deps = [dep for dep in missing_dep_out.split('\n')
+ if dep.startswith('*') and excluded_dep not in dep
+ ]
+ if missing_deps:
+ print(f'You need to install all modules on which {args.ec} depends first!')
+ print('\n\t'.join(['Missing:'] + missing_deps))
+ sys.exit(1)
+
+ # If the --ec argument is a (relative) existing path make it absolute so we can find it after the chdir
+ ec_arg = os.path.abspath(args.ec) if os.path.exists(args.ec) else args.ec
+ with temporary_directory() as tmp_dir:
+ old_dir = os.getcwd()
+ os.chdir(tmp_dir)
+ if args.verbose:
+ print('Running EasyBuild to get build environment')
+ run_shell_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment')
+ os.chdir(old_dir)
+
+ cmd = f"source {tmp_dir}/*.env && python {sys.argv[0]} '{args.package}'"
+ if args.verbose:
+ cmd += ' --verbose'
+ print('Restarting script in new build environment')
+
+ out = run_shell_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash')
+ print(out)
+ else:
+ if not can_run(sys.executable, '-m', 'venv', '-h'):
+ print("'venv' module not found. This should be available in Python 3.3+.")
+ sys.exit(1)
+ if 'PIP_PREFIX' in os.environ:
+ print("$PIP_PREFIX is set. Unsetting it as it doesn't work well with virtualenv.")
+ del os.environ['PIP_PREFIX']
+ os.environ['PYTHONNOUSERSITE'] = '1'
+ print_deps(args.package, args.verbose)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh
index d6631bfe0b..66b7e34bd8 100755
--- a/easybuild/scripts/findUpdatedEcs.sh
+++ b/easybuild/scripts/findUpdatedEcs.sh
@@ -33,51 +33,57 @@ function checkModule {
first_letter=${ec_filename:0:1}
letterPath=$easyconfigFolder/${first_letter,,}
if [[ -d "$letterPath" ]]; then
- ec_new="$(find "$letterPath" -type f -name "$ec_filename")"
+ ec_new="$(find "$letterPath" -type f -name "$ec_filename")"
else
- ec_new=
+ ec_new=
fi
# Fallback if not found
[[ -n "$ec_new" ]] || ec_new="$(find "$easyconfigFolder" -type f -name "$ec_filename")"
if [[ -z "$ec_new" ]]; then
- printError "=== Did not find new EC $ec_filename"
+ printError "=== Did not find new EC $ec_filename"
elif [[ ! -e "$ec_new" ]]; then
printError "=== Found multiple new ECs: $ec_new"
- elif ! out=$(diff "$ec_installed" "$ec_new"); then
- echo -e "${YELLOW}=== Needs updating: ${GREEN}${ec_installed}${YELLOW} vs ${GREEN}${ec_new}${NC}"
- if ((showDiff == 1)); then
- echo "$out"
+ elif ! out=$(diff -u "$ec_installed" "$ec_new"); then
+ if ((short == 1)); then
+ basename "$ec_installed"
+ else
+ echo -e "${YELLOW}=== Needs updating: ${GREEN}${ec_installed}${YELLOW} vs ${GREEN}${ec_new}${NC}"
+ if ((showDiff == 1)); then
+ echo "$out"
+ fi
fi
fi
}
ecDefaultFolder=
if path=$(which eb 2>/dev/null); then
- path=$(dirname "$path")
- for p in "$path" "$(dirname "$path")"; do
- if [ -d "$p/easybuild/easyconfigs" ]; then
- ecDefaultFolder=$p
- break
- fi
- done
+ path=$(dirname "$path")
+ for p in "$path" "$(dirname "$path")"; do
+ if [ -d "$p/easybuild/easyconfigs" ]; then
+ ecDefaultFolder=$p
+ break
+ fi
+ done
fi
function usage {
- echo "Usage: $(basename "$0") [--verbose] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER"
- echo
- echo "Check installed modules against the source EasyConfig (EC) files to determine which have changed."
- echo "Can either check the currently loaded modules or all modules installed in a specific location"
- echo
- echo "--verbose Verbose status output while checking"
- echo "--loaded Check only currently loaded modules"
- echo "--diff Show diff of changed module files"
- echo "--modules INSTALLPATH Check all modules in the specified (software) installpath, i.e. the root of module-binaries"
- echo "--easyconfigs EC-FOLDER Path to the folder containg the current/updated EasyConfigs. ${ecDefaultFolder:+Defaults to $ecDefaultFolder}"
- exit 0
+ echo "Usage: $(basename "$0") [--verbose] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER"
+ echo
+ echo "Check installed modules against the source EasyConfig (EC) files to determine which have changed."
+ echo "Can either check the currently loaded modules or all modules installed in a specific location"
+ echo
+ echo "--verbose Verbose status output while checking"
+ echo "--loaded Check only currently loaded modules"
+ echo "--short Only show filename of changed ECs"
+ echo "--diff Show diff of changed module files"
+ echo "--modules INSTALLPATH Check all modules in the specified (software) installpath, i.e. the root of module-binaries"
+ echo "--easyconfigs EC-FOLDER Path to the folder containg the current/updated EasyConfigs. ${ecDefaultFolder:+Defaults to $ecDefaultFolder}"
+ exit 0
}
checkLoadedModules=0
showDiff=0
+short=0
modulesFolder=""
easyconfigFolder=$ecDefaultFolder
@@ -89,6 +95,8 @@ while [[ $# -gt 0 ]]; do
verbose=1;;
-d|--diff)
showDiff=1;;
+ -s|--short)
+ short=1;;
-l|--loaded)
checkLoadedModules=1;;
-m|--modules)
diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py
index 388fb360d9..a14f18caf8 100755
--- a/easybuild/scripts/fix_docs.py
+++ b/easybuild/scripts/fix_docs.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
##
-# Copyright 2016-2023 Ghent University
+# Copyright 2016-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py
index 1cd63393d6..b8b554d0a9 100755
--- a/easybuild/scripts/mk_tmpl_easyblock_for.py
+++ b/easybuild/scripts/mk_tmpl_easyblock_for.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py
index 1bc87a1679..b477aa63d8 100755
--- a/easybuild/scripts/rpath_args.py
+++ b/easybuild/scripts/rpath_args.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
##
-# Copyright 2016-2023 Ghent University
+# Copyright 2016-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in
index 73eb21f0e0..5644a7ca75 100644
--- a/easybuild/scripts/rpath_wrapper_template.sh.in
+++ b/easybuild/scripts/rpath_wrapper_template.sh.in
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
##
-# Copyright 2016-2023 Ghent University
+# Copyright 2016-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/__init__.py b/easybuild/toolchains/__init__.py
index 309b1808cf..2f468f2812 100644
--- a/easybuild/toolchains/__init__.py
+++ b/easybuild/toolchains/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/cgmpich.py b/easybuild/toolchains/cgmpich.py
index ea04e20e75..9e25c6ec57 100644
--- a/easybuild/toolchains/cgmpich.py
+++ b/easybuild/toolchains/cgmpich.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/cgmpolf.py b/easybuild/toolchains/cgmpolf.py
index b335781374..5629ee5be3 100644
--- a/easybuild/toolchains/cgmpolf.py
+++ b/easybuild/toolchains/cgmpolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/cgmvapich2.py b/easybuild/toolchains/cgmvapich2.py
index e347cab483..76e306b12d 100644
--- a/easybuild/toolchains/cgmvapich2.py
+++ b/easybuild/toolchains/cgmvapich2.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/cgmvolf.py b/easybuild/toolchains/cgmvolf.py
index 1b9e122fcd..e68aa90fe1 100644
--- a/easybuild/toolchains/cgmvolf.py
+++ b/easybuild/toolchains/cgmvolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/cgompi.py b/easybuild/toolchains/cgompi.py
index 559f91f4ad..9b9267bf78 100644
--- a/easybuild/toolchains/cgompi.py
+++ b/easybuild/toolchains/cgompi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/cgoolf.py b/easybuild/toolchains/cgoolf.py
index c26e94b724..8e2ea8503e 100644
--- a/easybuild/toolchains/cgoolf.py
+++ b/easybuild/toolchains/cgoolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/clanggcc.py b/easybuild/toolchains/clanggcc.py
index 8537354ebe..c1e09e68ef 100644
--- a/easybuild/toolchains/clanggcc.py
+++ b/easybuild/toolchains/clanggcc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/compiler/__init__.py b/easybuild/toolchains/compiler/__init__.py
index 5933777f43..9c1ba469c5 100644
--- a/easybuild/toolchains/compiler/__init__.py
+++ b/easybuild/toolchains/compiler/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py
index df099fa659..1e925ec210 100644
--- a/easybuild/toolchains/compiler/clang.py
+++ b/easybuild/toolchains/compiler/clang.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
@@ -97,6 +97,7 @@ class Clang(Compiler):
}
# used with --optarch=GENERIC
COMPILER_GENERIC_OPTION = {
+ (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # default for -mabi is system-dependent
(systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic',
(systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic',
}
diff --git a/easybuild/toolchains/compiler/craype.py b/easybuild/toolchains/compiler/craype.py
index a0bec9f624..1d492c34db 100644
--- a/easybuild/toolchains/compiler/craype.py
+++ b/easybuild/toolchains/compiler/craype.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -65,7 +65,7 @@ class CrayPECompiler(Compiler):
COMPILER_UNIQUE_OPTS = {
'dynamic': (True, "Generate dynamically linked executable"),
- 'mpich-mt': (False, "Directs the driver to link in an alternate version of the Cray-MPICH library which \
+ 'mpich-mt': (False, "Directs the driver to link in an alternative version of the Cray-MPICH library which \
provides fine-grained multi-threading support to applications that perform \
MPI operations within threaded regions."),
'optarch': (False, "Enable architecture optimizations"),
diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py
index c87f1b21d1..3fd3d705f1 100644
--- a/easybuild/toolchains/compiler/cuda.py
+++ b/easybuild/toolchains/compiler/cuda.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py
index e7a861c1b7..437ec34048 100644
--- a/easybuild/toolchains/compiler/fujitsu.py
+++ b/easybuild/toolchains/compiler/fujitsu.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py
index 9ef98f5038..d8b9e901cf 100644
--- a/easybuild/toolchains/compiler/gcc.py
+++ b/easybuild/toolchains/compiler/gcc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -78,6 +78,15 @@ class Gcc(Compiler):
COMPILER_UNIQUE_OPTION_MAP['strict'] = no_recip_alternative
COMPILER_UNIQUE_OPTION_MAP['precise'] = no_recip_alternative
+ # gcc on RISC-V does not support -mno-recip, -mieee-fp, -mfno-math-errno...
+ # https://gcc.gnu.org/onlinedocs/gcc/RISC-V-Options.html
+ # there are no good alternatives, so stick to the default flags
+ if systemtools.get_cpu_family() == systemtools.RISCV:
+ COMPILER_UNIQUE_OPTION_MAP['strict'] = []
+ COMPILER_UNIQUE_OPTION_MAP['precise'] = []
+ COMPILER_UNIQUE_OPTION_MAP['loose'] = ['fno-math-errno']
+ COMPILER_UNIQUE_OPTION_MAP['veryloose'] = ['fno-math-errno']
+
# used when 'optarch' toolchain option is enabled (and --optarch is not specified)
COMPILER_OPTIMAL_ARCHITECTURE_OPTION = {
(systemtools.AARCH32, systemtools.ARM): 'mcpu=native', # implies -march=native and -mtune=native
@@ -94,6 +103,7 @@ class Gcc(Compiler):
(systemtools.AARCH64, systemtools.ARM): 'mcpu=generic', # implies -march=armv8-a and -mtune=generic
(systemtools.POWER, systemtools.POWER): 'mcpu=powerpc64', # no support for -march on POWER
(systemtools.POWER, systemtools.POWER_LE): 'mcpu=powerpc64le', # no support for -march on POWER
+ (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # default for -mabi is system-dependent
(systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic',
(systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic',
}
diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py
index ae97dfa87d..c33b5ee1a2 100644
--- a/easybuild/toolchains/compiler/intel_compilers.py
+++ b/easybuild/toolchains/compiler/intel_compilers.py
@@ -1,5 +1,5 @@
##
-# Copyright 2021-2023 Ghent University
+# Copyright 2021-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -48,7 +48,8 @@ class IntelCompilers(IntelIccIfort):
'oneapi': (None, "Use oneAPI compilers icx/icpx/ifx instead of classic compilers"),
'oneapi_c_cxx': (None, "Use oneAPI C/C++ compilers icx/icpx instead of classic Intel C/C++ compilers "
"(auto-enabled for Intel compilers version 2022.2.0, or newer)"),
- 'oneapi_fortran': (False, "Use oneAPI Fortran compiler ifx instead of classic Intel Fortran compiler"),
+ 'oneapi_fortran': (None, "Use oneAPI Fortran compiler ifx instead of classic Intel Fortran compiler "
+ "(auto-enabled for Intel compilers version 2024.0.0, or newer)"),
})
def _set_compiler_vars(self):
@@ -75,6 +76,9 @@ def set_variables(self):
# auto-enable use of oneAPI C/C++ compilers for sufficiently recent versions of Intel compilers
comp_ver = self.get_software_version(self.COMPILER_MODULE_NAME)[0]
if LooseVersion(comp_ver) >= LooseVersion('2022.2.0'):
+ if LooseVersion(comp_ver) >= LooseVersion('2024.0.0'):
+ if self.options.get('oneapi_fortran', None) is None:
+ self.options['oneapi_fortran'] = True
if self.options.get('oneapi_c_cxx', None) is None:
self.options['oneapi_c_cxx'] = True
diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py
index 1fdd04b42e..9287a0b056 100644
--- a/easybuild/toolchains/compiler/inteliccifort.py
+++ b/easybuild/toolchains/compiler/inteliccifort.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/compiler/nvhpc.py b/easybuild/toolchains/compiler/nvhpc.py
index 549c0e9a70..2011137716 100644
--- a/easybuild/toolchains/compiler/nvhpc.py
+++ b/easybuild/toolchains/compiler/nvhpc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015 Bart Oldeman
+# Copyright 2015-2024 Bart Oldeman
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/compiler/pgi.py b/easybuild/toolchains/compiler/pgi.py
index 77f04fe813..0f55614926 100644
--- a/easybuild/toolchains/compiler/pgi.py
+++ b/easybuild/toolchains/compiler/pgi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015 Bart Oldeman
+# Copyright 2015-2024 Bart Oldeman
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/compiler/systemcompiler.py b/easybuild/toolchains/compiler/systemcompiler.py
index 505f002d57..140ae3b159 100644
--- a/easybuild/toolchains/compiler/systemcompiler.py
+++ b/easybuild/toolchains/compiler/systemcompiler.py
@@ -1,5 +1,5 @@
##
-# Copyright 2019-2023 Ghent University
+# Copyright 2019-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,3 +40,10 @@ class SystemCompiler(Compiler):
"""System compiler"""
COMPILER_MODULE_NAME = []
COMPILER_FAMILY = TC_CONSTANT_SYSTEM
+
+ # The system compiler does not currently support even the shared options
+ # (changing this would require updating set_minimal_build_env() of the toolchain class)
+ COMPILER_UNIQUE_OPTS = None
+ COMPILER_SHARED_OPTS = None
+ COMPILER_UNIQUE_OPTION_MAP = None
+ COMPILER_SHARED_OPTION_MAP = None
diff --git a/easybuild/toolchains/craycce.py b/easybuild/toolchains/craycce.py
index cb9a4ff80e..13b92e55d2 100644
--- a/easybuild/toolchains/craycce.py
+++ b/easybuild/toolchains/craycce.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/craygnu.py b/easybuild/toolchains/craygnu.py
index 4e42d32f94..8d198362bd 100644
--- a/easybuild/toolchains/craygnu.py
+++ b/easybuild/toolchains/craygnu.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/crayintel.py b/easybuild/toolchains/crayintel.py
index 76cde0a8bc..5997eb63d6 100644
--- a/easybuild/toolchains/crayintel.py
+++ b/easybuild/toolchains/crayintel.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/craypgi.py b/easybuild/toolchains/craypgi.py
index 67e2bcec78..49004775a4 100644
--- a/easybuild/toolchains/craypgi.py
+++ b/easybuild/toolchains/craypgi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fcc.py b/easybuild/toolchains/fcc.py
index 9dedfd3a33..038b7540c3 100644
--- a/easybuild/toolchains/fcc.py
+++ b/easybuild/toolchains/fcc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/ffmpi.py b/easybuild/toolchains/ffmpi.py
index 34a56d229e..71a49f7913 100644
--- a/easybuild/toolchains/ffmpi.py
+++ b/easybuild/toolchains/ffmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fft/__init__.py b/easybuild/toolchains/fft/__init__.py
index 9915268575..afcf0211dd 100644
--- a/easybuild/toolchains/fft/__init__.py
+++ b/easybuild/toolchains/fft/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py
index 63d39cb658..9cb4c4e851 100644
--- a/easybuild/toolchains/fft/fftw.py
+++ b/easybuild/toolchains/fft/fftw.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fft/fujitsufftw.py b/easybuild/toolchains/fft/fujitsufftw.py
index 1e2ed28ec6..4de1769528 100644
--- a/easybuild/toolchains/fft/fujitsufftw.py
+++ b/easybuild/toolchains/fft/fujitsufftw.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py
index ad8f08ffd8..7882278a91 100644
--- a/easybuild/toolchains/fft/intelfftw.py
+++ b/easybuild/toolchains/fft/intelfftw.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py
index d5e57956a0..031daa4c9c 100644
--- a/easybuild/toolchains/foss.py
+++ b/easybuild/toolchains/foss.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fosscuda.py b/easybuild/toolchains/fosscuda.py
index a35c570781..9465515d36 100644
--- a/easybuild/toolchains/fosscuda.py
+++ b/easybuild/toolchains/fosscuda.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/fujitsu.py b/easybuild/toolchains/fujitsu.py
index ed858a84c7..707255e03c 100644
--- a/easybuild/toolchains/fujitsu.py
+++ b/easybuild/toolchains/fujitsu.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gcc.py b/easybuild/toolchains/gcc.py
index e359b8aa01..71091e795c 100644
--- a/easybuild/toolchains/gcc.py
+++ b/easybuild/toolchains/gcc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gcccore.py b/easybuild/toolchains/gcccore.py
index 87eaf4d1b6..ce4703e89c 100644
--- a/easybuild/toolchains/gcccore.py
+++ b/easybuild/toolchains/gcccore.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py
index fcf0981bac..47ab250824 100644
--- a/easybuild/toolchains/gcccuda.py
+++ b/easybuild/toolchains/gcccuda.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gfbf.py b/easybuild/toolchains/gfbf.py
index 9a7f279e6d..d41f5b7d06 100644
--- a/easybuild/toolchains/gfbf.py
+++ b/easybuild/toolchains/gfbf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2021-2023 Ghent University
+# Copyright 2021-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gimkl.py b/easybuild/toolchains/gimkl.py
index 8505cf5b87..1d7b9704f6 100644
--- a/easybuild/toolchains/gimkl.py
+++ b/easybuild/toolchains/gimkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py
index 6ac9788749..76e6bd638a 100644
--- a/easybuild/toolchains/gimpi.py
+++ b/easybuild/toolchains/gimpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gimpic.py b/easybuild/toolchains/gimpic.py
index 01a28c78ee..dde1d6c8f8 100644
--- a/easybuild/toolchains/gimpic.py
+++ b/easybuild/toolchains/gimpic.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/giolf.py b/easybuild/toolchains/giolf.py
index 64b99f2a5c..121c81121b 100644
--- a/easybuild/toolchains/giolf.py
+++ b/easybuild/toolchains/giolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/giolfc.py b/easybuild/toolchains/giolfc.py
index 3ca3111542..df798626aa 100644
--- a/easybuild/toolchains/giolfc.py
+++ b/easybuild/toolchains/giolfc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmacml.py b/easybuild/toolchains/gmacml.py
index 5561f85286..ee95c3f7f3 100644
--- a/easybuild/toolchains/gmacml.py
+++ b/easybuild/toolchains/gmacml.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmkl.py b/easybuild/toolchains/gmkl.py
index a58a05c488..67e33e1e62 100644
--- a/easybuild/toolchains/gmkl.py
+++ b/easybuild/toolchains/gmkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmklc.py b/easybuild/toolchains/gmklc.py
index b7ecd19115..bead896d5e 100644
--- a/easybuild/toolchains/gmklc.py
+++ b/easybuild/toolchains/gmklc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmpflf.py b/easybuild/toolchains/gmpflf.py
new file mode 100644
index 0000000000..30d10a90f7
--- /dev/null
+++ b/easybuild/toolchains/gmpflf.py
@@ -0,0 +1,45 @@
+##
+# Copyright 2013-2024 Ghent University
+#
+# This file is triple-licensed under GPLv2 (see below), MIT, and
+# BSD three-clause licenses.
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+EasyBuild support for gmpflf compiler toolchain (includes GCC, MPICH, FlexiBLAS, LAPACK, ScaLAPACK and FFTW).
+
+Authors:
+
+* Richard Topouchian (University of Bergen)
+"""
+from easybuild.toolchains.gmpich import Gmpich
+from easybuild.toolchains.gfbf import Gfbf
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.flexiblas import FlexiBLAS
+from easybuild.toolchains.linalg.scalapack import ScaLAPACK
+
+
+class Gmpflf(Gmpich, FlexiBLAS, ScaLAPACK, Fftw):
+ """Compiler toolchain with GCC, MPICH, FlexiBLAS, ScaLAPACK and FFTW."""
+ NAME = 'gmpflf'
+ SUBTOOLCHAIN = [Gmpich.NAME, Gfbf.NAME]
diff --git a/easybuild/toolchains/gmpich.py b/easybuild/toolchains/gmpich.py
index 941421e50e..66d2b29123 100644
--- a/easybuild/toolchains/gmpich.py
+++ b/easybuild/toolchains/gmpich.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmpich2.py b/easybuild/toolchains/gmpich2.py
index bad4e0bbaf..f734d3cb56 100644
--- a/easybuild/toolchains/gmpich2.py
+++ b/easybuild/toolchains/gmpich2.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmpit.py b/easybuild/toolchains/gmpit.py
index 395940a639..a39baf934d 100644
--- a/easybuild/toolchains/gmpit.py
+++ b/easybuild/toolchains/gmpit.py
@@ -1,5 +1,5 @@
##
-# Copyright 2022-2023 Ghent University
+# Copyright 2022-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmpolf.py b/easybuild/toolchains/gmpolf.py
index 2bca2b44dd..c5a8be97d9 100644
--- a/easybuild/toolchains/gmpolf.py
+++ b/easybuild/toolchains/gmpolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/gmvapich2.py b/easybuild/toolchains/gmvapich2.py
index 46ebb4d489..26aa6ad73e 100644
--- a/easybuild/toolchains/gmvapich2.py
+++ b/easybuild/toolchains/gmvapich2.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gmvolf.py b/easybuild/toolchains/gmvolf.py
index 50c943cde6..613d8a6ed3 100644
--- a/easybuild/toolchains/gmvolf.py
+++ b/easybuild/toolchains/gmvolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/gnu.py b/easybuild/toolchains/gnu.py
index a16eed38bd..84b06af535 100644
--- a/easybuild/toolchains/gnu.py
+++ b/easybuild/toolchains/gnu.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/goalf.py b/easybuild/toolchains/goalf.py
index d833e3c1c5..db1280907b 100644
--- a/easybuild/toolchains/goalf.py
+++ b/easybuild/toolchains/goalf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gobff.py b/easybuild/toolchains/gobff.py
index 2cbb366dfe..9df26ff706 100644
--- a/easybuild/toolchains/gobff.py
+++ b/easybuild/toolchains/gobff.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/goblf.py b/easybuild/toolchains/goblf.py
index 618d813149..63e7c0c0a4 100644
--- a/easybuild/toolchains/goblf.py
+++ b/easybuild/toolchains/goblf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gofbf.py b/easybuild/toolchains/gofbf.py
index 71269a57df..3128c87091 100644
--- a/easybuild/toolchains/gofbf.py
+++ b/easybuild/toolchains/gofbf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2021-2023 Ghent University
+# Copyright 2021-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/golf.py b/easybuild/toolchains/golf.py
index c318ec9fee..6d046413a8 100644
--- a/easybuild/toolchains/golf.py
+++ b/easybuild/toolchains/golf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/golfc.py b/easybuild/toolchains/golfc.py
index bdce5ddcec..3b99dce5c0 100644
--- a/easybuild/toolchains/golfc.py
+++ b/easybuild/toolchains/golfc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gomkl.py b/easybuild/toolchains/gomkl.py
index 663bc32daa..ba93882a1b 100644
--- a/easybuild/toolchains/gomkl.py
+++ b/easybuild/toolchains/gomkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gomklc.py b/easybuild/toolchains/gomklc.py
index fa19da4b50..95620d8360 100644
--- a/easybuild/toolchains/gomklc.py
+++ b/easybuild/toolchains/gomklc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gompi.py b/easybuild/toolchains/gompi.py
index 35db8a8330..7af87123c9 100644
--- a/easybuild/toolchains/gompi.py
+++ b/easybuild/toolchains/gompi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gompic.py b/easybuild/toolchains/gompic.py
index a9f08dcf2a..0beb2b4d10 100644
--- a/easybuild/toolchains/gompic.py
+++ b/easybuild/toolchains/gompic.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/goolf.py b/easybuild/toolchains/goolf.py
index 6577cb31c4..099f7bc11a 100644
--- a/easybuild/toolchains/goolf.py
+++ b/easybuild/toolchains/goolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/goolfc.py b/easybuild/toolchains/goolfc.py
index 26cd18ec9b..76cf4db890 100644
--- a/easybuild/toolchains/goolfc.py
+++ b/easybuild/toolchains/goolfc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gpsmpi.py b/easybuild/toolchains/gpsmpi.py
index c6a7fd04b8..529d17dc5e 100644
--- a/easybuild/toolchains/gpsmpi.py
+++ b/easybuild/toolchains/gpsmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gpsolf.py b/easybuild/toolchains/gpsolf.py
index 1414178019..e87f3a5de7 100644
--- a/easybuild/toolchains/gpsolf.py
+++ b/easybuild/toolchains/gpsolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/gqacml.py b/easybuild/toolchains/gqacml.py
index 5533bd1f3a..7456b4011e 100644
--- a/easybuild/toolchains/gqacml.py
+++ b/easybuild/toolchains/gqacml.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gsmpi.py b/easybuild/toolchains/gsmpi.py
index 865dbc813b..68f1a6d0f0 100644
--- a/easybuild/toolchains/gsmpi.py
+++ b/easybuild/toolchains/gsmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/gsolf.py b/easybuild/toolchains/gsolf.py
index 5d30493bb1..cb917ca7d3 100644
--- a/easybuild/toolchains/gsolf.py
+++ b/easybuild/toolchains/gsolf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iccifort.py b/easybuild/toolchains/iccifort.py
index d1b6804a93..38ec224a5b 100644
--- a/easybuild/toolchains/iccifort.py
+++ b/easybuild/toolchains/iccifort.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iccifortcuda.py b/easybuild/toolchains/iccifortcuda.py
index 5f024dc8ce..149cfc8e39 100644
--- a/easybuild/toolchains/iccifortcuda.py
+++ b/easybuild/toolchains/iccifortcuda.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/ictce.py b/easybuild/toolchains/ictce.py
index 265f249306..2055346b37 100644
--- a/easybuild/toolchains/ictce.py
+++ b/easybuild/toolchains/ictce.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/ifbf.py b/easybuild/toolchains/ifbf.py
index 3507e1eab9..9f689279cb 100644
--- a/easybuild/toolchains/ifbf.py
+++ b/easybuild/toolchains/ifbf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iibff.py b/easybuild/toolchains/iibff.py
index 9fe2b2b0c8..07a6b1e567 100644
--- a/easybuild/toolchains/iibff.py
+++ b/easybuild/toolchains/iibff.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iimkl.py b/easybuild/toolchains/iimkl.py
index 9d3d6a97f3..605445cc5c 100644
--- a/easybuild/toolchains/iimkl.py
+++ b/easybuild/toolchains/iimkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iimklc.py b/easybuild/toolchains/iimklc.py
index e9508c95ad..89325c2e67 100644
--- a/easybuild/toolchains/iimklc.py
+++ b/easybuild/toolchains/iimklc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py
index 00d9745a42..9337365ec6 100644
--- a/easybuild/toolchains/iimpi.py
+++ b/easybuild/toolchains/iimpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iimpic.py b/easybuild/toolchains/iimpic.py
index 53ca437150..45a8df3804 100644
--- a/easybuild/toolchains/iimpic.py
+++ b/easybuild/toolchains/iimpic.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iiqmpi.py b/easybuild/toolchains/iiqmpi.py
index aee09bb658..5ff1466ff3 100644
--- a/easybuild/toolchains/iiqmpi.py
+++ b/easybuild/toolchains/iiqmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/impich.py b/easybuild/toolchains/impich.py
index 2b7fba07f9..7feda33805 100644
--- a/easybuild/toolchains/impich.py
+++ b/easybuild/toolchains/impich.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/impmkl.py b/easybuild/toolchains/impmkl.py
index 1653b31d5c..952bff34ce 100644
--- a/easybuild/toolchains/impmkl.py
+++ b/easybuild/toolchains/impmkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/intel-para.py b/easybuild/toolchains/intel-para.py
index a9d4ca0313..ddfee3aaae 100644
--- a/easybuild/toolchains/intel-para.py
+++ b/easybuild/toolchains/intel-para.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py
index 7dd625588f..659b05faf2 100644
--- a/easybuild/toolchains/intel.py
+++ b/easybuild/toolchains/intel.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/intel_compilers.py b/easybuild/toolchains/intel_compilers.py
index ab5de03386..27fdfbebfd 100644
--- a/easybuild/toolchains/intel_compilers.py
+++ b/easybuild/toolchains/intel_compilers.py
@@ -1,5 +1,5 @@
##
-# Copyright 2021-2023 Ghent University
+# Copyright 2021-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/intelcuda.py b/easybuild/toolchains/intelcuda.py
index 343715e43b..bdd59abaab 100644
--- a/easybuild/toolchains/intelcuda.py
+++ b/easybuild/toolchains/intelcuda.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iofbf.py b/easybuild/toolchains/iofbf.py
index 3410ffaf9f..cd3313d600 100644
--- a/easybuild/toolchains/iofbf.py
+++ b/easybuild/toolchains/iofbf.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iomkl.py b/easybuild/toolchains/iomkl.py
index cefe83c3b1..033277919c 100644
--- a/easybuild/toolchains/iomkl.py
+++ b/easybuild/toolchains/iomkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iomklc.py b/easybuild/toolchains/iomklc.py
index 102fa7a311..f40cd78e16 100644
--- a/easybuild/toolchains/iomklc.py
+++ b/easybuild/toolchains/iomklc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py
index 318db78481..b578c45892 100644
--- a/easybuild/toolchains/iompi.py
+++ b/easybuild/toolchains/iompi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iompic.py b/easybuild/toolchains/iompic.py
index 1bcb2eac71..5b675f5621 100644
--- a/easybuild/toolchains/iompic.py
+++ b/easybuild/toolchains/iompic.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/ipsmpi.py b/easybuild/toolchains/ipsmpi.py
index 1a381414c3..a8a772adcb 100644
--- a/easybuild/toolchains/ipsmpi.py
+++ b/easybuild/toolchains/ipsmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/iqacml.py b/easybuild/toolchains/iqacml.py
index fc63968583..e4c5476cd4 100644
--- a/easybuild/toolchains/iqacml.py
+++ b/easybuild/toolchains/iqacml.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/ismkl.py b/easybuild/toolchains/ismkl.py
index 90ff100eeb..4ccdc87054 100644
--- a/easybuild/toolchains/ismkl.py
+++ b/easybuild/toolchains/ismkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/__init__.py b/easybuild/toolchains/linalg/__init__.py
index 152bae1e3d..90cdb068a8 100644
--- a/easybuild/toolchains/linalg/__init__.py
+++ b/easybuild/toolchains/linalg/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py
index 47cb13cfe1..1470b1f360 100644
--- a/easybuild/toolchains/linalg/acml.py
+++ b/easybuild/toolchains/linalg/acml.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/atlas.py b/easybuild/toolchains/linalg/atlas.py
index 4d53f5bd32..bc894bb108 100644
--- a/easybuild/toolchains/linalg/atlas.py
+++ b/easybuild/toolchains/linalg/atlas.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/blacs.py b/easybuild/toolchains/linalg/blacs.py
index a3963e9daa..201681c51f 100644
--- a/easybuild/toolchains/linalg/blacs.py
+++ b/easybuild/toolchains/linalg/blacs.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/blis.py b/easybuild/toolchains/linalg/blis.py
index fa607d0527..d89f3b8d83 100644
--- a/easybuild/toolchains/linalg/blis.py
+++ b/easybuild/toolchains/linalg/blis.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/flame.py b/easybuild/toolchains/linalg/flame.py
index 78ac8f1ca4..5f6f5cfe87 100644
--- a/easybuild/toolchains/linalg/flame.py
+++ b/easybuild/toolchains/linalg/flame.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py
index 3726e7444f..b994383bb5 100644
--- a/easybuild/toolchains/linalg/flexiblas.py
+++ b/easybuild/toolchains/linalg/flexiblas.py
@@ -1,5 +1,5 @@
##
-# Copyright 2021-2023 Ghent University
+# Copyright 2021-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/fujitsussl.py b/easybuild/toolchains/linalg/fujitsussl.py
index ecaff56683..dde792455b 100644
--- a/easybuild/toolchains/linalg/fujitsussl.py
+++ b/easybuild/toolchains/linalg/fujitsussl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/gotoblas.py b/easybuild/toolchains/linalg/gotoblas.py
index 17d98413a8..e8a99a651a 100644
--- a/easybuild/toolchains/linalg/gotoblas.py
+++ b/easybuild/toolchains/linalg/gotoblas.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py
index 39238654cd..a4a16973f0 100644
--- a/easybuild/toolchains/linalg/intelmkl.py
+++ b/easybuild/toolchains/linalg/intelmkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -139,6 +139,7 @@ def _set_blas_variables(self):
self.variables.nappend_el('CFLAGS', 'DMKL_ILP64')
# exact paths/linking statements depend on imkl version
+ root = self.get_software_root(self.BLAS_MODULE_NAME)[0]
found_version = self.get_software_version(self.BLAS_MODULE_NAME)[0]
ver = LooseVersion(found_version)
if ver < LooseVersion('10.3'):
@@ -146,6 +147,8 @@ def _set_blas_variables(self):
self.BLAS_INCLUDE_DIR = ['include']
else:
if ver >= LooseVersion('2021'):
+ if os.path.islink(os.path.join(root, 'mkl', 'latest')):
+ found_version = os.readlink(os.path.join(root, 'mkl', 'latest'))
basedir = os.path.join('mkl', found_version)
else:
basedir = 'mkl'
diff --git a/easybuild/toolchains/linalg/lapack.py b/easybuild/toolchains/linalg/lapack.py
index e9821a659d..e26c9e2961 100644
--- a/easybuild/toolchains/linalg/lapack.py
+++ b/easybuild/toolchains/linalg/lapack.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/libsci.py b/easybuild/toolchains/linalg/libsci.py
index 87c42116f5..6d7e93c16d 100644
--- a/easybuild/toolchains/linalg/libsci.py
+++ b/easybuild/toolchains/linalg/libsci.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -65,13 +65,20 @@ def _get_software_root(self, name, required=True):
"""Get install prefix for specified software name; special treatment for Cray modules."""
if name == 'cray-libsci':
# Cray-provided LibSci module
- env_var = 'CRAY_LIBSCI_PREFIX_DIR'
- root = os.getenv(env_var, None)
+ root = None
+ # consider both $CRAY_LIBSCI_PREFIX_DIR and $CRAY_PE_LIBSCI_PREFIX_DIR,
+ # cfr. https://github.com/easybuilders/easybuild-framework/issues/4536
+ env_vars = ('CRAY_LIBSCI_PREFIX_DIR', 'CRAY_PE_LIBSCI_PREFIX_DIR')
+ for env_var in env_vars:
+ root = os.getenv(env_var, None)
+ if root is not None:
+ self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root)
+ break
+
if root is None:
if required:
- raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var)
- else:
- self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root)
+ env_vars_str = ', '.join('$' + e for e in env_vars)
+ raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_vars_str)
else:
root = super(LibSci, self)._get_software_root(name, required=required)
diff --git a/easybuild/toolchains/linalg/openblas.py b/easybuild/toolchains/linalg/openblas.py
index e28d4f7dd2..18118d1f01 100644
--- a/easybuild/toolchains/linalg/openblas.py
+++ b/easybuild/toolchains/linalg/openblas.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/linalg/scalapack.py b/easybuild/toolchains/linalg/scalapack.py
index ef4d24ef30..980e3bfdab 100644
--- a/easybuild/toolchains/linalg/scalapack.py
+++ b/easybuild/toolchains/linalg/scalapack.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/__init__.py b/easybuild/toolchains/mpi/__init__.py
index aea95e9051..c5abd0f595 100644
--- a/easybuild/toolchains/mpi/__init__.py
+++ b/easybuild/toolchains/mpi/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/craympich.py b/easybuild/toolchains/mpi/craympich.py
index 4c62c45519..8eab736d82 100644
--- a/easybuild/toolchains/mpi/craympich.py
+++ b/easybuild/toolchains/mpi/craympich.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -53,7 +53,7 @@ class CrayMPICH(Mpi):
MPI_COMPILER_MPIFC = CrayPECompiler.COMPILER_FC
# no MPI wrappers, so no need to specify serial compiler
- MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES])
+ MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES}
def _set_mpi_compiler_variables(self):
"""Set the MPI compiler variables"""
diff --git a/easybuild/toolchains/mpi/fujitsumpi.py b/easybuild/toolchains/mpi/fujitsumpi.py
index f81e1d66f0..ad65c463cf 100644
--- a/easybuild/toolchains/mpi/fujitsumpi.py
+++ b/easybuild/toolchains/mpi/fujitsumpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -45,7 +45,7 @@ class FujitsuMPI(Mpi):
MPI_TYPE = TC_CONSTANT_MPI_TYPE_OPENMPI
# OpenMPI reads from CC etc env variables
- MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES])
+ MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES}
MPI_LINK_INFO_OPTION = '-showme:link'
diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py
index 6b9f86430a..8bc67fe788 100644
--- a/easybuild/toolchains/mpi/intelmpi.py
+++ b/easybuild/toolchains/mpi/intelmpi.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/mpich.py b/easybuild/toolchains/mpi/mpich.py
index b0c42e57d8..7a067a0d37 100644
--- a/easybuild/toolchains/mpi/mpich.py
+++ b/easybuild/toolchains/mpi/mpich.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -56,7 +56,7 @@ class Mpich(Mpi):
MPI_COMPILER_MPIFC = None
# clear MPI wrapper command options
- MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES])
+ MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES}
def _set_mpi_compiler_variables(self):
"""Set the MPICH_{CC, CXX, F77, F90, FC} variables."""
diff --git a/easybuild/toolchains/mpi/mpich2.py b/easybuild/toolchains/mpi/mpich2.py
index bd85d30532..b6f1f6f082 100644
--- a/easybuild/toolchains/mpi/mpich2.py
+++ b/easybuild/toolchains/mpi/mpich2.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py
index 5af054661c..43202b59d3 100644
--- a/easybuild/toolchains/mpi/mpitrampoline.py
+++ b/easybuild/toolchains/mpi/mpitrampoline.py
@@ -1,5 +1,5 @@
##
-# Copyright 2022-2023 Ghent University
+# Copyright 2022-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -53,7 +53,7 @@ class MPItrampoline(Mpi):
MPI_COMPILER_MPIFC = None
# MPItrampoline reads from CC etc env variables
- MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES])
+ MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES}
MPI_LINK_INFO_OPTION = '-showme:link'
diff --git a/easybuild/toolchains/mpi/mvapich2.py b/easybuild/toolchains/mpi/mvapich2.py
index afb2602f44..4f69c970f5 100644
--- a/easybuild/toolchains/mpi/mvapich2.py
+++ b/easybuild/toolchains/mpi/mvapich2.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py
index 630262f48b..91d10923e7 100644
--- a/easybuild/toolchains/mpi/openmpi.py
+++ b/easybuild/toolchains/mpi/openmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -61,7 +61,7 @@ class OpenMPI(Mpi):
MPI_COMPILER_MPIFC = None
# OpenMPI reads from CC etc env variables
- MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES])
+ MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES}
MPI_LINK_INFO_OPTION = '-showme:link'
diff --git a/easybuild/toolchains/mpi/psmpi.py b/easybuild/toolchains/mpi/psmpi.py
index cfbf720a74..07e8c9af78 100644
--- a/easybuild/toolchains/mpi/psmpi.py
+++ b/easybuild/toolchains/mpi/psmpi.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/qlogicmpi.py b/easybuild/toolchains/mpi/qlogicmpi.py
index 26c0663756..b7355cd3e1 100644
--- a/easybuild/toolchains/mpi/qlogicmpi.py
+++ b/easybuild/toolchains/mpi/qlogicmpi.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/mpi/spectrummpi.py b/easybuild/toolchains/mpi/spectrummpi.py
index e0d4b2f530..1fee1d51b7 100644
--- a/easybuild/toolchains/mpi/spectrummpi.py
+++ b/easybuild/toolchains/mpi/spectrummpi.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/nvhpc.py b/easybuild/toolchains/nvhpc.py
index aca69d386f..b4b871f84f 100644
--- a/easybuild/toolchains/nvhpc.py
+++ b/easybuild/toolchains/nvhpc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015 Bart Oldeman
+# Copyright 2015-2024 Bart Oldeman
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/nvofbf.py b/easybuild/toolchains/nvofbf.py
index 41e89e5a32..62d3cb4d37 100644
--- a/easybuild/toolchains/nvofbf.py
+++ b/easybuild/toolchains/nvofbf.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/nvompi.py b/easybuild/toolchains/nvompi.py
index 5f2e25f03d..3feedaae35 100644
--- a/easybuild/toolchains/nvompi.py
+++ b/easybuild/toolchains/nvompi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2021 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/nvompic.py b/easybuild/toolchains/nvompic.py
index e4ac0d6106..b9c5abc298 100644
--- a/easybuild/toolchains/nvompic.py
+++ b/easybuild/toolchains/nvompic.py
@@ -1,6 +1,6 @@
##
-# Copyright 2016-2023 Ghent University
-# Copyright 2016-2023 Forschungszentrum Juelich
+# Copyright 2016-2024 Ghent University
+# Copyright 2016-2024 Forschungszentrum Juelich
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/nvpsmpi.py b/easybuild/toolchains/nvpsmpi.py
index 9e102c9d53..cbaee42bc0 100644
--- a/easybuild/toolchains/nvpsmpi.py
+++ b/easybuild/toolchains/nvpsmpi.py
@@ -1,6 +1,6 @@
##
-# Copyright 2016-2021 Ghent University
-# Copyright 2016-2021 Forschungszentrum Juelich
+# Copyright 2016-2024 Ghent University
+# Copyright 2016-2024 Forschungszentrum Juelich
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/nvpsmpic.py b/easybuild/toolchains/nvpsmpic.py
index 604630ff8e..0b916c30a0 100644
--- a/easybuild/toolchains/nvpsmpic.py
+++ b/easybuild/toolchains/nvpsmpic.py
@@ -1,6 +1,6 @@
##
-# Copyright 2016-2023 Ghent University
-# Copyright 2016-2023 Forschungszentrum Juelich
+# Copyright 2016-2024 Ghent University
+# Copyright 2016-2024 Forschungszentrum Juelich
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/pgi.py b/easybuild/toolchains/pgi.py
index 2ee3dc3839..df2258bda4 100644
--- a/easybuild/toolchains/pgi.py
+++ b/easybuild/toolchains/pgi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015 Bart Oldeman
+# Copyright 2015-2024 Bart Oldeman
#
# This file is triple-licensed under GPLv2 (see below), MIT, and
# BSD three-clause licenses.
diff --git a/easybuild/toolchains/pmkl.py b/easybuild/toolchains/pmkl.py
index a4ad73d7cd..8b22d1d35f 100644
--- a/easybuild/toolchains/pmkl.py
+++ b/easybuild/toolchains/pmkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/pomkl.py b/easybuild/toolchains/pomkl.py
index ea85e2d440..f28f445296 100644
--- a/easybuild/toolchains/pomkl.py
+++ b/easybuild/toolchains/pomkl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/pompi.py b/easybuild/toolchains/pompi.py
index f8a9e00039..9de2e52050 100644
--- a/easybuild/toolchains/pompi.py
+++ b/easybuild/toolchains/pompi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/toolchains/system.py b/easybuild/toolchains/system.py
index cfb46a93b2..8a8b17cfaa 100644
--- a/easybuild/toolchains/system.py
+++ b/easybuild/toolchains/system.py
@@ -1,5 +1,5 @@
##
-# Copyright 2019-2023 Ghent University
+# Copyright 2019-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py
index dee4fc0d12..8968a12699 100644
--- a/easybuild/tools/__init__.py
+++ b/easybuild/tools/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -37,14 +37,4 @@
__path__ = __import__('pkgutil').extend_path(__path__, __name__)
-import distutils.version
-import warnings
from easybuild.tools.loose_version import LooseVersion # noqa(F401)
-
-
-class StrictVersion(distutils.version.StrictVersion):
- """Temporary wrapper over distuitls StrictVersion that silences the deprecation warning"""
- def __init__(self, *args, **kwargs):
- with warnings.catch_warnings():
- warnings.simplefilter("ignore", category=DeprecationWarning)
- distutils.version.StrictVersion.__init__(self, *args, **kwargs)
diff --git a/easybuild/tools/asyncprocess.py b/easybuild/tools/asyncprocess.py
index b3a7d330ff..0a3c133fc4 100644
--- a/easybuild/tools/asyncprocess.py
+++ b/easybuild/tools/asyncprocess.py
@@ -1,6 +1,6 @@
##
# Copyright 2005 Josiah Carlson
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# The Asynchronous Python Subprocess recipe was originally created by Josiah Carlson.
# and released under the GPL v2 on March 14, 2012
diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py
index 9cd2f0c88c..f5ce1ebc01 100644
--- a/easybuild/tools/build_details.py
+++ b/easybuild/tools/build_details.py
@@ -1,4 +1,4 @@
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py
index f8529fff57..fac06ecbbc 100644
--- a/easybuild/tools/build_log.py
+++ b/easybuild/tools/build_log.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,12 +40,12 @@
import tempfile
from copy import copy
from datetime import datetime
+from enum import IntEnum
from easybuild.base import fancylogger
from easybuild.base.exceptions import LoggedException
from easybuild.tools.version import VERSION, this_is_easybuild
-
# EasyBuild message prefix
EB_MSG_PREFIX = "=="
@@ -55,17 +55,71 @@
# allow some experimental experimental code
EXPERIMENTAL = False
-DEPRECATED_DOC_URL = 'http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html'
+DEPRECATED_DOC_URL = 'https://docs.easybuild.io/deprecated-functionality/'
DRY_RUN_BUILD_DIR = None
DRY_RUN_SOFTWARE_INSTALL_DIR = None
DRY_RUN_MODULES_INSTALL_DIR = None
+CWD_NOTFOUND_ERROR = (
+ "Current working directory does not exist! It was either unexpectedly removed "
+ "by an external process to EasyBuild or the filesystem is misbehaving."
+)
+
DEVEL_LOG_LEVEL = logging.DEBUG - 1
logging.addLevelName(DEVEL_LOG_LEVEL, 'DEVEL')
+class EasyBuildExit(IntEnum):
+ """
+ Table of exit codes
+ """
+ SUCCESS = 0
+ ERROR = 1
+ # core errors
+ OPTION_ERROR = 2
+ VALUE_ERROR = 3
+ MISSING_EASYCONFIG = 4
+ EASYCONFIG_ERROR = 5
+ MISSING_EASYBLOCK = 6
+ EASYBLOCK_ERROR = 7
+ MODULE_ERROR = 8
+ # step errors in order of execution
+ FAIL_FETCH_STEP = 10
+ FAIL_READY_STEP = 11
+ FAIL_SOURCE_STEP = 12
+ FAIL_PATCH_STEP = 13
+ FAIL_PREPARE_STEP = 14
+ FAIL_CONFIGURE_STEP = 15
+ FAIL_BUILD_STEP = 16
+ FAIL_TEST_STEP = 17
+ FAIL_INSTALL_STEP = 18
+ FAIL_EXTENSIONS_STEP = 19
+ FAIL_POST_ITER_STEP = 20
+ FAIL_POST_PROC_STEP = 21
+ FAIL_SANITY_CHECK_STEP = 22
+ FAIL_CLEANUP_STEP = 23
+ FAIL_MODULE_STEP = 24
+ FAIL_PERMISSIONS_STEP = 25
+ FAIL_PACKAGE_STEP = 26
+ FAIL_TEST_CASES_STEP = 27
+ # errors on missing things
+ MISSING_SOURCES = 30
+ MISSING_DEPENDENCY = 31
+ MISSING_SYSTEM_DEPENDENCY = 32
+ MISSING_EB_DEPENDENCY = 33
+ # errors on specific task failures
+ FAIL_SYSTEM_CHECK = 40
+ FAIL_DOWNLOAD = 41
+ FAIL_CHECKSUM = 42
+ FAIL_EXTRACT = 43
+ FAIL_PATCH_APPLY = 44
+ FAIL_SANITY_CHECK = 45
+ FAIL_MODULE_WRITE = 46
+ FAIL_GITHUB = 47
+
+
class EasyBuildError(LoggedException):
"""
EasyBuildError is thrown when EasyBuild runs into something horribly wrong.
@@ -75,12 +129,13 @@ class EasyBuildError(LoggedException):
# always include location where error was raised from, even under 'python -O'
INCLUDE_LOCATION = True
- def __init__(self, msg, *args):
+ def __init__(self, msg, *args, exit_code=EasyBuildExit.ERROR, **kwargs):
"""Constructor: initialise EasyBuildError instance."""
if args:
msg = msg % args
- LoggedException.__init__(self, msg)
+ LoggedException.__init__(self, msg, exit_code=exit_code, **kwargs)
self.msg = msg
+ self.exit_code = exit_code
def __str__(self):
"""Return string representation of this EasyBuildError instance."""
diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index ecd503ba1a..08c6cbf776 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -50,7 +50,7 @@
from easybuild.base import fancylogger
from easybuild.base.frozendict import FrozenDictKnownKeys
from easybuild.base.wrapper import create_base_metaclass
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit
try:
import rich # noqa
@@ -96,7 +96,7 @@
DEFAULT_ENV_FOR_SHEBANG = '/usr/bin/env'
DEFAULT_ENVVAR_USERS_MODULES = 'HOME'
DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60 # 1 week (in seconds)
-DEFAULT_JOB_BACKEND = 'GC3Pie'
+DEFAULT_JOB_BACKEND = 'Slurm'
DEFAULT_JOB_EB_CMD = 'eb'
DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log")
DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5
@@ -121,6 +121,8 @@
DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders'
DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild")
DEFAULT_REPOSITORY = 'FileRepository'
+EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io'
+DEFAULT_EXTRA_SOURCE_URLS = (EASYBUILD_SOURCES_URL,)
# Filter these CUDA libraries by default from the RPATH sanity check.
# These are the only four libraries for which the CUDA toolkit ships stubs. By design, one is supposed to build
# against the stub versions, but use the libraries that come with the CUDA driver at runtime. That means they should
@@ -173,6 +175,10 @@
OUTPUT_STYLE_RICH = 'rich'
OUTPUT_STYLES = (OUTPUT_STYLE_AUTO, OUTPUT_STYLE_BASIC, OUTPUT_STYLE_NO_COLOR, OUTPUT_STYLE_RICH)
+PYTHONPATH = 'PYTHONPATH'
+EBPYTHONPREFIXES = 'EBPYTHONPREFIXES'
+PYTHON_SEARCH_PATH_TYPES = [PYTHONPATH, EBPYTHONPREFIXES]
+
class Singleton(ABCMeta):
"""Serves as metaclass for classes that should implement the Singleton pattern.
@@ -221,6 +227,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'filter_env_vars',
'filter_rpath_sanity_libs',
'force_download',
+ 'from_commit',
'git_working_dirs_path',
'github_user',
'github_org',
@@ -230,6 +237,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'http_header_fields_urlpat',
'hooks',
'ignore_dirs',
+ 'include_easyblocks_from_commit',
'insecure_download',
'job_backend_config',
'job_cores',
@@ -258,6 +266,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'rpath_override_dirs',
'required_linked_shared_libs',
'skip',
+ 'software_commit',
'stop',
'subdir_user_modules',
'sysroot',
@@ -275,6 +284,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'debug',
'debug_lmod',
'dump_autopep8',
+ 'dump_env_script',
'enforce_checksums',
'experimental',
'extended_dry_run',
@@ -290,7 +300,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'install_latest_eb_release',
'logtostdout',
'minimal_toolchains',
- 'module_extensions',
'module_only',
'package',
'parallel_extensions_install',
@@ -304,9 +313,11 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'set_gid_bit',
'silence_hook_trigger',
'skip_extensions',
+ 'skip_sanity_check',
'skip_test_cases',
'skip_test_step',
'sticky_bit',
+ 'terse',
'unit_testing_mode',
'upload_test_report',
'update_modules_tool_cache',
@@ -325,10 +336,12 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'lib64_fallback_sanity_check',
'lib64_lib_symlink',
'map_toolchains',
+ 'module_extensions',
'modules_tool_version_check',
'mpi_tests',
'pre_create_installdir',
'show_progress_bar',
+ 'strict_rpath_sanity_check',
'trace',
],
EMPTY_LIST: [
@@ -387,6 +400,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'defaultopt': [
'default_opt_level',
],
+ DEFAULT_EXTRA_SOURCE_URLS: [
+ 'extra_source_urls',
+ ],
DEFAULT_ALLOW_LOADED_MODULES: [
'allow_loaded_modules',
],
@@ -396,6 +412,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
OUTPUT_STYLE_AUTO: [
'output_style',
],
+ PYTHONPATH: [
+ 'prefer_python_search_path',
+ ]
}
# build option that do not have a perfectly matching command line option
BUILD_OPTIONS_OTHER = {
@@ -403,14 +422,14 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'build_specs',
'command_line',
'external_modules_metadata',
- 'pr_paths',
+ 'extra_ec_paths',
+ 'mod_depends_on', # deprecated
'robot_path',
'valid_module_classes',
'valid_stops',
],
False: [
'dry_run',
- 'mod_depends_on',
'recursive_mod_unload',
'retain_all_deps',
'silent',
@@ -495,7 +514,10 @@ def get_items_check_required(self):
"""
missing = [x for x in self.KNOWN_KEYS if x not in self]
if len(missing) > 0:
- raise EasyBuildError("Cannot determine value for configuration variables %s. Please specify it.", missing)
+ raise EasyBuildError(
+ "Cannot determine value for configuration variables %s. Please specify it.", ', '.join(missing),
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
return self.items()
@@ -528,7 +550,10 @@ def init(options, config_options_dict):
tmpdict['sourcepath'] = sourcepath.split(':')
_log.debug("Converted source path ('%s') to a list of paths: %s" % (sourcepath, tmpdict['sourcepath']))
elif not isinstance(sourcepath, (tuple, list)):
- raise EasyBuildError("Value for sourcepath has invalid type (%s): %s", type(sourcepath), sourcepath)
+ raise EasyBuildError(
+ "Value for sourcepath has invalid type (%s): %s", type(sourcepath), sourcepath,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
# initialize configuration variables (any future calls to ConfigurationVariables() will yield the same instance
variables = ConfigurationVariables(tmpdict, ignore_unknown_keys=True)
@@ -570,7 +595,7 @@ def init_build_options(build_options=None, cmdline_options=None):
cmdline_options.ignore_osdeps = True
cmdline_build_option_names = [k for ks in BUILD_OPTIONS_CMDLINE.values() for k in ks]
- active_build_options.update(dict([(key, getattr(cmdline_options, key)) for key in cmdline_build_option_names]))
+ active_build_options.update({key: getattr(cmdline_options, key) for key in cmdline_build_option_names})
# other options which can be derived but have no perfectly matching cmdline option
active_build_options.update({
'check_osdeps': not cmdline_options.ignore_osdeps,
@@ -588,12 +613,12 @@ def init_build_options(build_options=None, cmdline_options=None):
# seed in defaults to make sure all build options are defined, and that build_option() doesn't fail on valid keys
bo = {}
for build_options_by_default in [BUILD_OPTIONS_CMDLINE, BUILD_OPTIONS_OTHER]:
- for default in build_options_by_default:
+ for default, options in build_options_by_default.items():
if default == EMPTY_LIST:
- for opt in build_options_by_default[default]:
+ for opt in options:
bo[opt] = []
else:
- bo.update(dict([(opt, default) for opt in build_options_by_default[default]]))
+ bo.update({opt: default for opt in options})
bo.update(active_build_options)
# BuildOptions is a singleton, so any future calls to BuildOptions will yield the same instance
@@ -612,7 +637,7 @@ def build_option(key, **kwargs):
error_msg = "Undefined build option: '%s'. " % key
error_msg += "Make sure you have set up the EasyBuild configuration using set_up_configuration() "
error_msg += "(from easybuild.tools.options) in case you're not using EasyBuild via the 'eb' CLI."
- raise EasyBuildError(error_msg)
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.OPTION_ERROR)
def update_build_option(key, value):
@@ -677,7 +702,10 @@ def install_path(typ=None):
known_types = ['modules', 'software']
if typ not in known_types:
- raise EasyBuildError("Unknown type specified in install_path(): %s (known: %s)", typ, ', '.join(known_types))
+ raise EasyBuildError(
+ "Unknown type specified in install_path(): %s (known: %s)", typ, ', '.join(known_types),
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
variables = ConfigurationVariables()
@@ -730,7 +758,7 @@ def container_path():
def get_modules_tool():
"""
- Return modules tool (EnvironmentModulesC, Lmod, ...)
+ Return modules tool (EnvironmentModules, Lmod, ...)
"""
# 'modules_tool' key will only be present if EasyBuild config is initialized
return ConfigurationVariables().get('modules_tool', None)
@@ -769,7 +797,10 @@ def get_output_style():
output_style = OUTPUT_STYLE_BASIC
if output_style == OUTPUT_STYLE_RICH and not HAVE_RICH:
- raise EasyBuildError("Can't use '%s' output style, Rich Python package is not available!", OUTPUT_STYLE_RICH)
+ raise EasyBuildError(
+ "Can't use '%s' output style, Rich Python package is not available!", OUTPUT_STYLE_RICH,
+ exit_code=EasyBuildExit.MISSING_EB_DEPENDENCY
+ )
return output_style
@@ -794,8 +825,10 @@ def log_file_format(return_directory=False, ec=None, date=None, timestamp=None):
logfile_format = ConfigurationVariables()['logfile_format']
if not isinstance(logfile_format, tuple) or len(logfile_format) != 2:
- raise EasyBuildError("Incorrect log file format specification, should be 2-tuple (, ): %s",
- logfile_format)
+ raise EasyBuildError(
+ "Incorrect log file format specification, should be 2-tuple (, ): %s", logfile_format,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
idx = int(not return_directory)
res = ConfigurationVariables()['logfile_format'][idx] % {
@@ -902,7 +935,10 @@ def find_last_log(curlog):
sorted_paths = [p for (_, p) in sorted(paths)]
except OSError as err:
- raise EasyBuildError("Failed to locate/select/order log files matching '%s': %s", glob_pattern, err)
+ raise EasyBuildError(
+ "Failed to locate/select/order log files matching '%s': %s", glob_pattern, err,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
try:
# log of current session is typically listed last, should be taken into account
diff --git a/easybuild/tools/containers/__init__.py b/easybuild/tools/containers/__init__.py
index 76cb37219a..215ba61c0c 100644
--- a/easybuild/tools/containers/__init__.py
+++ b/easybuild/tools/containers/__init__.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/containers/apptainer.py b/easybuild/tools/containers/apptainer.py
index 8a294ce1ad..c8c5c059f9 100644
--- a/easybuild/tools/containers/apptainer.py
+++ b/easybuild/tools/containers/apptainer.py
@@ -1,4 +1,4 @@
-# Copyright 2022-2023 Ghent University
+# Copyright 2022-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -29,7 +29,7 @@
import os
import re
-from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg
from easybuild.tools.containers.singularity import SingularityContainer
from easybuild.tools.config import CONT_IMAGE_FORMAT_EXT3, CONT_IMAGE_FORMAT_SANDBOX
from easybuild.tools.config import CONT_IMAGE_FORMAT_SIF, CONT_IMAGE_FORMAT_SQUASHFS
@@ -49,7 +49,7 @@ def apptainer_version():
"""Get Apptainer version."""
version_cmd = "apptainer --version"
res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True)
- if res.exit_code:
+ if res.exit_code != EasyBuildExit.SUCCESS:
raise EasyBuildError(f"Error running '{version_cmd}': {res.output}")
regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip())
diff --git a/easybuild/tools/containers/base.py b/easybuild/tools/containers/base.py
index 68a49610d3..faf3b0cc7a 100644
--- a/easybuild/tools/containers/base.py
+++ b/easybuild/tools/containers/base.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/containers/common.py b/easybuild/tools/containers/common.py
index bcbdce9a9b..6824c16864 100644
--- a/easybuild/tools/containers/common.py
+++ b/easybuild/tools/containers/common.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py
index 5a1597d634..1308ac57db 100644
--- a/easybuild/tools/containers/docker.py
+++ b/easybuild/tools/containers/docker.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py
index 638a985ae5..19151a9c78 100644
--- a/easybuild/tools/containers/singularity.py
+++ b/easybuild/tools/containers/singularity.py
@@ -1,4 +1,4 @@
-# Copyright 2017-2023 Ghent University
+# Copyright 2017-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -34,7 +34,7 @@
import re
from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg
from easybuild.tools.config import CONT_IMAGE_FORMAT_EXT3, CONT_IMAGE_FORMAT_SANDBOX
from easybuild.tools.config import CONT_IMAGE_FORMAT_SIF, CONT_IMAGE_FORMAT_SQUASHFS
from easybuild.tools.config import build_option, container_path
@@ -163,7 +163,7 @@ def singularity_version():
"""Get Singularity version."""
version_cmd = "singularity --version"
res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True)
- if res.exit_code:
+ if res.exit_code != EasyBuildExit.SUCCESS:
raise EasyBuildError(f"Error running '{version_cmd}': {res.output}")
regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip())
diff --git a/easybuild/tools/containers/utils.py b/easybuild/tools/containers/utils.py
index d543ca77fc..c36af49a8f 100644
--- a/easybuild/tools/containers/utils.py
+++ b/easybuild/tools/containers/utils.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -34,7 +34,7 @@
from functools import reduce
from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg
from easybuild.tools.filetools import which
from easybuild.tools.run import run_shell_cmd
@@ -77,7 +77,7 @@ def check_tool(tool_name, min_tool_version=None):
version_cmd = f"{tool_name} --version"
res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True)
- if res.exit_code:
+ if res.exit_code != EasyBuildExit.SUCCESS:
raise EasyBuildError(f"Error running '{version_cmd}' for tool {tool_name} with output: {res.output}")
regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip())
diff --git a/easybuild/tools/convert.py b/easybuild/tools/convert.py
index 9a675850f9..98d3b63711 100644
--- a/easybuild/tools/convert.py
+++ b/easybuild/tools/convert.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py
index 5a41090166..bcee3917e6 100644
--- a/easybuild/tools/docs.py
+++ b/easybuild/tools/docs.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -50,7 +50,7 @@
from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, process_easyconfig
from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
-from easybuild.framework.easyconfig.parser import EasyConfigParser
+from easybuild.framework.easyconfig.parser import ALTERNATIVE_EASYCONFIG_PARAMETERS, EasyConfigParser
from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_DYNAMIC
from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_NAMES_EASYCONFIG
from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_LOWER, TEMPLATE_NAMES_LOWER_TEMPLATE
@@ -315,7 +315,7 @@ def avail_easyconfig_licenses_md():
return '\n'.join(doc)
-def avail_easyconfig_params_md(title, grouped_params):
+def avail_easyconfig_params_md(title, grouped_params, alternative_params):
"""
Compose overview of available easyconfig parameters, in MarkDown format.
"""
@@ -328,13 +328,14 @@ def avail_easyconfig_params_md(title, grouped_params):
for grpname in grouped_params:
# group section title
title = "%s%s parameters" % (grpname[0].upper(), grpname[1:])
- table_titles = ["**Parameter name**", "**Description**", "**Default value**"]
+ table_titles = ["**Parameter name**", "**Description**", "**Default value**", "**Alternative name**"]
keys = sorted(grouped_params[grpname].keys())
values = [grouped_params[grpname][key] for key in keys]
table_values = [
['`%s`' % name for name in keys], # parameter name
[x[0].replace('<', '<').replace('>', '>') for x in values], # description
- ['`' + str(quote_str(x[1])) + '`' for x in values] # default value
+ ['`' + str(quote_str(x[1])) + '`' for x in values], # default value
+ ['`%s`' % alternative_params[name] if name in alternative_params else '' for name in keys],
]
doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2))
@@ -343,7 +344,7 @@ def avail_easyconfig_params_md(title, grouped_params):
return '\n'.join(doc)
-def avail_easyconfig_params_rst(title, grouped_params):
+def avail_easyconfig_params_rst(title, grouped_params, alternative_params):
"""
Compose overview of available easyconfig parameters, in RST format.
"""
@@ -357,13 +358,14 @@ def avail_easyconfig_params_rst(title, grouped_params):
for grpname in grouped_params:
# group section title
title = "%s parameters" % grpname
- table_titles = ["**Parameter name**", "**Description**", "**Default value**"]
+ table_titles = ["**Parameter name**", "**Description**", "**Default value**", "**Alternative name**"]
keys = sorted(grouped_params[grpname].keys())
values = [grouped_params[grpname][key] for key in keys]
table_values = [
['``%s``' % name for name in keys], # parameter name
[x[0] for x in values], # description
- [str(quote_str(x[1])) for x in values] # default value
+ [str(quote_str(x[1])) for x in values], # default value
+ ['``%s``' % alternative_params[name] if name in alternative_params else '' for name in keys],
]
doc.extend(rst_title_and_table(title, table_titles, table_values))
@@ -372,14 +374,14 @@ def avail_easyconfig_params_rst(title, grouped_params):
return '\n'.join(doc)
-def avail_easyconfig_params_json():
+def avail_easyconfig_params_json(*args):
"""
Compose overview of available easyconfig parameters, in json format.
"""
raise NotImplementedError("JSON output format not supported for avail_easyconfig_params_json")
-def avail_easyconfig_params_txt(title, grouped_params):
+def avail_easyconfig_params_txt(title, grouped_params, alternative_params):
"""
Compose overview of available easyconfig parameters, in plain text format.
"""
@@ -399,7 +401,17 @@ def avail_easyconfig_params_txt(title, grouped_params):
# line by parameter
for name, (descr, dflt) in sorted(grouped_params[grpname].items()):
- doc.append("{0:<{nw}} {1:} [default: {2:}]".format(name, descr, str(quote_str(dflt)), nw=nw))
+ line = ' '.join([
+ '{0:<{nw}} ',
+ '{1:}',
+ '[default: {2:}]',
+ ]).format(name, descr, str(quote_str(dflt)), nw=nw)
+
+ alternative = alternative_params.get(name)
+ if alternative:
+ line += ' {alternative: %s}' % alternative
+
+ doc.append(line)
doc.append('')
return '\n'.join(doc)
@@ -418,6 +430,9 @@ def avail_easyconfig_params(easyblock, output_format=FORMAT_TXT):
extra_params = app.extra_options()
params.update(extra_params)
+ # reverse mapping of alternative easyconfig parameter names
+ alternative_params = {v: k for k, v in ALTERNATIVE_EASYCONFIG_PARAMETERS.items()}
+
# compose title
title = "Available easyconfig parameters"
if extra_params:
@@ -443,7 +458,7 @@ def avail_easyconfig_params(easyblock, output_format=FORMAT_TXT):
del grouped_params[grpname]
# compose output, according to specified format (txt, rst, ...)
- return generate_doc('avail_easyconfig_params_%s' % output_format, [title, grouped_params])
+ return generate_doc('avail_easyconfig_params_%s' % output_format, [title, grouped_params, alternative_params])
def avail_easyconfig_templates(output_format=FORMAT_TXT):
@@ -463,16 +478,16 @@ def avail_easyconfig_templates_txt():
# step 1: add TEMPLATE_NAMES_EASYCONFIG
doc.append('Template names/values derived from easyconfig instance')
- for name in TEMPLATE_NAMES_EASYCONFIG:
- doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1]))
+ for name, curDoc in TEMPLATE_NAMES_EASYCONFIG.items():
+ doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, curDoc))
doc.append('')
# step 2: add SOFTWARE_VERSIONS
doc.append('Template names/values for (short) software versions')
- for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
- doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, pref, name))
- doc.append("%s%%(%sshortver)s: short version for %s (.)" % (INDENT_4SPACES, pref, name))
- doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, pref, name))
+ for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items():
+ doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, prefix, name))
+ doc.append("%s%%(%sshortver)s: short version for %s (.)" % (INDENT_4SPACES, prefix, name))
+ doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, prefix, name))
doc.append('')
# step 3: add remaining config
@@ -491,20 +506,20 @@ def avail_easyconfig_templates_txt():
# step 5: template_values can/should be updated from outside easyconfig
# (eg the run_step code in EasyBlock)
doc.append('Template values set outside EasyBlock runstep')
- for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
- doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1]))
+ for name, cur_doc in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.items():
+ doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, cur_doc))
doc.append('')
# some template values are only defined dynamically,
# see template_constant_dict function in easybuild.framework.easyconfigs.templates
doc.append('Template values which are defined dynamically')
- for name in TEMPLATE_NAMES_DYNAMIC:
- doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1]))
+ for name, cur_doc in TEMPLATE_NAMES_DYNAMIC.items():
+ doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, cur_doc))
doc.append('')
doc.append('Template constants that can be used in easyconfigs')
- for cst in TEMPLATE_CONSTANTS:
- doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, cst[0], cst[2], cst[1]))
+ for name, (value, cur_doc) in TEMPLATE_CONSTANTS.items():
+ doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, name, cur_doc, value))
return '\n'.join(doc)
@@ -515,8 +530,8 @@ def avail_easyconfig_templates_rst():
title = 'Template names/values derived from easyconfig instance'
table_values = [
- ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYCONFIG],
- [name[1] for name in TEMPLATE_NAMES_EASYCONFIG],
+ ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYCONFIG],
+ list(TEMPLATE_NAMES_EASYCONFIG.values()),
]
doc = rst_title_and_table(title, table_titles, table_values)
doc.append('')
@@ -524,10 +539,10 @@ def avail_easyconfig_templates_rst():
title = 'Template names/values for (short) software versions'
ver = []
ver_desc = []
- for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
- ver.append('``%%(%smajver)s``' % pref)
- ver.append('``%%(%sshortver)s``' % pref)
- ver.append('``%%(%sver)s``' % pref)
+ for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items():
+ ver.append('``%%(%smajver)s``' % prefix)
+ ver.append('``%%(%sshortver)s``' % prefix)
+ ver.append('``%%(%sver)s``' % prefix)
ver_desc.append('major version for %s' % name)
ver_desc.append('short version for %s (.)' % name)
ver_desc.append('full version for %s' % name)
@@ -550,24 +565,24 @@ def avail_easyconfig_templates_rst():
title = 'Template values set outside EasyBlock runstep'
table_values = [
- ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
- [name[1] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
+ ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
+ list(TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.values()),
]
doc.extend(rst_title_and_table(title, table_titles, table_values))
title = 'Template values which are defined dynamically'
table_values = [
- ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC],
- [name[1] for name in TEMPLATE_NAMES_DYNAMIC],
+ ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_DYNAMIC],
+ list(TEMPLATE_NAMES_DYNAMIC.values()),
]
doc.extend(rst_title_and_table(title, table_titles, table_values))
title = 'Template constants that can be used in easyconfigs'
- titles = ['Constant', 'Template value', 'Template name']
+ titles = ['Constant', 'Template description', 'Template value']
table_values = [
- ['``%s``' % cst[0] for cst in TEMPLATE_CONSTANTS],
- [cst[2] for cst in TEMPLATE_CONSTANTS],
- ['``%s``' % cst[1] for cst in TEMPLATE_CONSTANTS],
+ ['``%s``' % name for name in TEMPLATE_CONSTANTS],
+ [doc for _, doc in TEMPLATE_CONSTANTS.values()],
+ ['``%s``' % value for value, _ in TEMPLATE_CONSTANTS.values()],
]
doc.extend(rst_title_and_table(title, titles, table_values))
@@ -580,8 +595,8 @@ def avail_easyconfig_templates_md():
title = 'Template names/values derived from easyconfig instance'
table_values = [
- ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYCONFIG],
- [name[1] for name in TEMPLATE_NAMES_EASYCONFIG],
+ ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYCONFIG],
+ list(TEMPLATE_NAMES_EASYCONFIG.values()),
]
doc = md_title_and_table(title, table_titles, table_values, title_level=2)
doc.append('')
@@ -589,10 +604,10 @@ def avail_easyconfig_templates_md():
title = 'Template names/values for (short) software versions'
ver = []
ver_desc = []
- for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
- ver.append('``%%(%smajver)s``' % pref)
- ver.append('``%%(%sshortver)s``' % pref)
- ver.append('``%%(%sver)s``' % pref)
+ for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items():
+ ver.append('``%%(%smajver)s``' % prefix)
+ ver.append('``%%(%sshortver)s``' % prefix)
+ ver.append('``%%(%sver)s``' % prefix)
ver_desc.append('major version for %s' % name)
ver_desc.append('short version for %s (``.``)' % name)
ver_desc.append('full version for %s' % name)
@@ -616,27 +631,28 @@ def avail_easyconfig_templates_md():
title = 'Template values set outside EasyBlock runstep'
table_values = [
- ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
- [name[1] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
+ ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP],
+ list(TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.values()),
]
doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2))
doc.append('')
title = 'Template values which are defined dynamically'
table_values = [
- ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC],
- [name[1] for name in TEMPLATE_NAMES_DYNAMIC],
+ ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_DYNAMIC],
+ list(TEMPLATE_NAMES_DYNAMIC.values()),
]
doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2))
doc.append('')
title = 'Template constants that can be used in easyconfigs'
- titles = ['Constant', 'Template value', 'Template name']
+ titles = ['Constant', 'Template description', 'Template value']
table_values = [
- ['``%s``' % cst[0] for cst in TEMPLATE_CONSTANTS],
- [cst[2] for cst in TEMPLATE_CONSTANTS],
- ['``%s``' % cst[1] for cst in TEMPLATE_CONSTANTS],
+ ['``%s``' % name for name in TEMPLATE_CONSTANTS],
+ [doc for _, doc in TEMPLATE_CONSTANTS.values()],
+ ['``%s``' % value for value, _ in TEMPLATE_CONSTANTS.values()],
]
+
doc.extend(md_title_and_table(title, titles, table_values, title_level=2))
return '\n'.join(doc)
@@ -789,7 +805,7 @@ def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False
if isinstance(ec, dict):
template_values = template_constant_dict(ec)
for key in keys:
- if '%(' in info[key]:
+ if info[key] and '%(' in info[key]:
try:
info[key] = info[key] % template_values
except (KeyError, TypeError, ValueError) as err:
@@ -807,8 +823,8 @@ def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False
# rebuild software, only retain entries with a corresponding available module
software, all_software = {}, software
- for key in all_software:
- for entry in all_software[key]:
+ for key, entries in all_software.items():
+ for entry in entries:
if entry['mod_name'] in avail_mod_names:
software.setdefault(key, []).append(entry)
@@ -1092,11 +1108,9 @@ def list_toolchains(output_format=FORMAT_TXT):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain('')
- all_tcs = [x for x in all_tcs]
- all_tcs_names = [x.NAME for x in all_tcs]
-
# start with dict that maps toolchain name to corresponding subclass of Toolchain
- tcs = dict(zip(all_tcs_names, all_tcs))
+ # filter deprecated 'dummy' toolchain
+ tcs = {tc.NAME: tc for tc in all_tcs}
for tcname in sorted(tcs):
tcc = tcs[tcname]
diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py
index d68755ff24..07f9ef6e50 100644
--- a/easybuild/tools/environment.py
+++ b/easybuild/tools/environment.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -54,8 +54,8 @@ def write_changes(filename):
"""
try:
with open(filename, 'w') as script:
- for key in _changes:
- script.write('export %s=%s\n' % (key, shell_quote(_changes[key])))
+ for key, changed_value in _changes.items():
+ script.write('export %s=%s\n' % (key, shell_quote(changed_value)))
except IOError as err:
raise EasyBuildError("Failed to write to %s: %s", filename, err)
reset_changes()
@@ -83,9 +83,9 @@ def setvar(key, value, verbose=True):
:param verbose: include message in dry run output for defining this environment variable
"""
- if key in os.environ:
+ try:
oldval_info = "previous value: '%s'" % os.environ[key]
- else:
+ except KeyError:
oldval_info = "previously undefined"
# os.putenv() is not necessary. os.environ will call this.
os.environ[key] = value
@@ -136,7 +136,7 @@ def read_environment(env_vars, strict=False):
:param env_vars: a dict with key a name, value a environment variable name
:param strict: boolean, if True enforces that all specified environment variables are found
"""
- result = dict([(k, os.environ.get(v)) for k, v in env_vars.items() if v in os.environ])
+ result = {k: os.environ.get(v) for k, v in env_vars.items() if v in os.environ}
if not len(env_vars) == len(result):
missing = ','.join(["%s / %s" % (k, v) for k, v in env_vars.items() if k not in result])
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 52118304fe..89613e0f03 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -42,11 +42,13 @@
"""
import datetime
import difflib
+import filecmp
import glob
import hashlib
import inspect
import itertools
import os
+import platform
import re
import shutil
import signal
@@ -61,8 +63,10 @@
import urllib.request as std_urllib
from easybuild.base import fancylogger
+from easybuild.tools import LooseVersion
# import build_log must stay, to use of EasyBuildLog
-from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, CWD_NOTFOUND_ERROR
+from easybuild.tools.build_log import dry_run_msg, print_msg, print_warning
from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path
from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
from easybuild.tools.hooks import load_source
@@ -121,13 +125,25 @@
CHECKSUM_TYPE_MD5 = 'md5'
CHECKSUM_TYPE_SHA256 = 'sha256'
-DEFAULT_CHECKSUM = CHECKSUM_TYPE_MD5
+DEFAULT_CHECKSUM = CHECKSUM_TYPE_SHA256
+
+
+def _hashlib_md5():
+ """
+ Wrapper function for hashlib.md5,
+ to set usedforsecurity to False when supported (Python >= 3.9)
+ """
+ kwargs = {}
+ if sys.version_info[0] >= 3 and sys.version_info[1] >= 9:
+ kwargs = {'usedforsecurity': False}
+ return hashlib.md5(**kwargs)
+
# map of checksum types to checksum functions
CHECKSUM_FUNCTIONS = {
'adler32': lambda p: calc_block_checksum(p, ZlibChecksum(zlib.adler32)),
'crc32': lambda p: calc_block_checksum(p, ZlibChecksum(zlib.crc32)),
- CHECKSUM_TYPE_MD5: lambda p: calc_block_checksum(p, hashlib.md5()),
+ CHECKSUM_TYPE_MD5: lambda p: calc_block_checksum(p, _hashlib_md5()),
'sha1': lambda p: calc_block_checksum(p, hashlib.sha1()),
CHECKSUM_TYPE_SHA256: lambda p: calc_block_checksum(p, hashlib.sha256()),
'sha512': lambda p: calc_block_checksum(p, hashlib.sha512()),
@@ -407,6 +423,22 @@ def remove(paths):
raise EasyBuildError("Specified path to remove is not an existing file or directory: %s", path)
+def get_cwd(must_exist=True):
+ """
+ Retrieve current working directory
+ """
+ try:
+ cwd = os.getcwd()
+ except FileNotFoundError as err:
+ if must_exist is True:
+ raise EasyBuildError(CWD_NOTFOUND_ERROR)
+
+ _log.debug("Failed to determine current working directory, but proceeding anyway: %s", err)
+ cwd = None
+
+ return cwd
+
+
def change_dir(path):
"""
Change to directory at specified location.
@@ -414,19 +446,19 @@ def change_dir(path):
:param path: location to change to
:return: previous location we were in
"""
- # determining the current working directory can fail if we're in a non-existing directory
- try:
- cwd = os.getcwd()
- except OSError as err:
- _log.debug("Failed to determine current working directory (but proceeding anyway: %s", err)
- cwd = None
+ # determine origin working directory: can fail if non-existent
+ prev_dir = get_cwd(must_exist=False)
try:
os.chdir(path)
except OSError as err:
- raise EasyBuildError("Failed to change from %s to %s: %s", cwd, path, err)
+ raise EasyBuildError("Failed to change from %s to %s: %s", prev_dir, path, err)
- return cwd
+ # determine final working directory: must exist
+ # stoplight meant to catch filesystems in a faulty state
+ get_cwd()
+
+ return prev_dir
def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced=False, change_into_dir=False,
@@ -570,11 +602,11 @@ def normalize_path(path):
def is_alt_pypi_url(url):
- """Determine whether specified URL is already an alternate PyPI URL, i.e. whether it contains a hash."""
+ """Determine whether specified URL is already an alternative PyPI URL, i.e. whether it contains a hash."""
# example: .../packages/5b/03/e135b19fadeb9b1ccb45eac9f60ca2dc3afe72d099f6bd84e03cb131f9bf/easybuild-2.7.0.tar.gz
alt_url_regex = re.compile('/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/[^/]+$')
res = bool(alt_url_regex.search(url))
- _log.debug("Checking whether '%s' is an alternate PyPI URL using pattern '%s'...: %s",
+ _log.debug("Checking whether '%s' is an alternative PyPI URL using pattern '%s'...: %s",
url, alt_url_regex.pattern, res)
return res
@@ -622,7 +654,7 @@ def handle_starttag(self, tag, attrs):
def derive_alt_pypi_url(url):
- """Derive alternate PyPI URL for given URL."""
+ """Derive alternative PyPI URL for given URL."""
alt_pypi_url = None
# example input URL: https://pypi.python.org/packages/source/e/easybuild/easybuild-2.7.0.tar.gz
@@ -671,9 +703,9 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header
if argline == '' or '#' in argline[0]:
continue # permit comment lines: ignore them
- if os.path.isfile(os.path.join(os.getcwd(), argline)):
+ if os.path.isfile(os.path.join(get_cwd(), argline)):
# expand existing relative path to absolute
- argline = os.path.join(os.path.join(os.getcwd(), argline))
+ argline = os.path.join(os.path.join(get_cwd(), argline))
if os.path.isfile(argline):
# argline is a file path, so read that instead
_log.debug('File included in parse_http_header_fields_urlpat: %s' % argline)
@@ -826,7 +858,10 @@ def download_file(filename, url, path, forced=False, trace=True):
if error_re.match(str(err)):
switch_to_requests = True
except Exception as err:
- raise EasyBuildError("Unexpected error occurred when trying to download %s to %s: %s", url, path, err)
+ raise EasyBuildError(
+ "Unexpected error occurred when trying to download %s to %s: %s", url, path, err,
+ exit_code=EasyBuildExit.FAIL_DOWNLOAD
+ )
if not downloaded and attempt_cnt < max_attempts:
_log.info("Attempt %d of downloading %s to %s failed, trying again..." % (attempt_cnt, url, path))
@@ -954,11 +989,12 @@ def load_index(path, ignore_dirs=None):
# check whether index is still valid
if valid_ts:
curr_ts = datetime.datetime.now()
+ terse = build_option('terse')
if curr_ts > valid_ts:
- print_warning("Index for %s is no longer valid (too old), so ignoring it...", path)
+ print_warning("Index for %s is no longer valid (too old), so ignoring it...", path, silent=terse)
index = None
else:
- print_msg("found valid index for %s, so using it...", path)
+ print_msg("found valid index for %s, so using it...", path, silent=terse)
return index or None
@@ -1040,7 +1076,10 @@ def locate_files(files, paths, ignore_subdirs=None):
if files_to_find:
filenames = ', '.join([f for (_, f) in files_to_find])
paths = ', '.join(paths)
- raise EasyBuildError("One or more files not found: %s (search paths: %s)", filenames, paths)
+ raise EasyBuildError(
+ "One or more files not found: %s (search paths: %s)", filenames, paths,
+ exit_code=EasyBuildExit.MISSING_EASYCONFIG
+ )
return [os.path.abspath(f) for f in files]
@@ -1191,12 +1230,16 @@ def compute_checksum(path, checksum_type=DEFAULT_CHECKSUM):
Compute checksum of specified file.
:param path: Path of file to compute checksum for
- :param checksum_type: type(s) of checksum ('adler32', 'crc32', 'md5' (default), 'sha1', 'sha256', 'sha512', 'size')
+ :param checksum_type: type(s) of checksum ('adler32', 'crc32', 'md5', 'sha1', 'sha256', 'sha512', 'size')
"""
if checksum_type not in CHECKSUM_FUNCTIONS:
raise EasyBuildError("Unknown checksum type (%s), supported types are: %s",
checksum_type, CHECKSUM_FUNCTIONS.keys())
+ if checksum_type in ['adler32', 'crc32', 'md5', 'sha1', 'size']:
+ _log.deprecated("Checksum type %s is deprecated. Use sha256 (default) or sha512 instead" % checksum_type,
+ '6.0')
+
try:
checksum = CHECKSUM_FUNCTIONS[checksum_type](path)
except IOError as err:
@@ -1229,12 +1272,15 @@ def calc_block_checksum(path, algorithm):
return algorithm.hexdigest()
-def verify_checksum(path, checksums):
+def verify_checksum(path, checksums, computed_checksums=None):
"""
Verify checksum of specified file.
:param path: path of file to verify checksum of
- :param checksums: checksum values (and type, optionally, default is MD5), e.g., 'af314', ('sha', '5ec1b')
+ :param checksums: checksum values (and type, optionally, default is sha256), e.g., 'af314', ('sha', '5ec1b')
+ :param computed_checksums: Optional dictionary of (current) checksum(s) for this file
+ indexed by the checksum type (e.g. 'sha256').
+ Each existing entry will be used, missing ones will be computed.
"""
filename = os.path.basename(path)
@@ -1286,12 +1332,18 @@ def verify_checksum(path, checksums):
# no matching checksums
return False
else:
- raise EasyBuildError("Invalid checksum spec '%s': should be a string (MD5 or SHA256), "
+ raise EasyBuildError("Invalid checksum spec '%s': should be a string (SHA256), "
"2-tuple (type, value), or tuple of alternative checksum specs.",
checksum)
- actual_checksum = compute_checksum(path, typ)
- _log.debug("Computed %s checksum for %s: %s (correct checksum: %s)" % (typ, path, actual_checksum, checksum))
+ if computed_checksums is not None and typ in computed_checksums:
+ actual_checksum = computed_checksums[typ]
+ computed_str = 'Precomputed'
+ else:
+ actual_checksum = compute_checksum(path, typ)
+ computed_str = 'Computed'
+ _log.debug("%s %s checksum for %s: %s (correct checksum: %s)" %
+ (computed_str, typ, path, actual_checksum, checksum))
if actual_checksum != checksum:
return False
@@ -1327,14 +1379,14 @@ def get_local_dirs_purged():
# and hidden directories
ignoredirs = ["easybuild"]
- lst = os.listdir(os.getcwd())
+ lst = os.listdir(get_cwd())
lst = [d for d in lst if not d.startswith('.') and d not in ignoredirs]
return lst
lst = get_local_dirs_purged()
- new_dir = os.getcwd()
+ new_dir = get_cwd()
while len(lst) == 1:
- new_dir = os.path.join(os.getcwd(), lst[0])
+ new_dir = os.path.join(get_cwd(), lst[0])
if not os.path.isdir(new_dir):
break
@@ -1482,8 +1534,10 @@ def create_patch_info(patch_spec):
else:
patch_info['copy'] = patch_arg
else:
- raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element",
- str(patch_spec))
+ raise EasyBuildError(
+ "Wrong patch spec '%s', only int/string are supported as 2nd element", str(patch_spec),
+ exit_code=EasyBuildExit.EASYCONFIG_ERROR
+ )
elif isinstance(patch_spec, str):
validate_patch_spec(patch_spec)
@@ -1494,13 +1548,17 @@ def create_patch_info(patch_spec):
if key in valid_keys:
patch_info[key] = patch_spec[key]
else:
- raise EasyBuildError("Wrong patch spec '%s', use of unknown key %s in dict (valid keys are %s)",
- str(patch_spec), key, valid_keys)
+ raise EasyBuildError(
+ "Wrong patch spec '%s', use of unknown key %s in dict (valid keys are %s)",
+ str(patch_spec), key, valid_keys, exit_code=EasyBuildExit.EASYCONFIG_ERROR
+ )
# Dict must contain at least the patchfile name
if 'name' not in patch_info.keys():
- raise EasyBuildError("Wrong patch spec '%s', when using a dict 'name' entry must be supplied",
- str(patch_spec))
+ raise EasyBuildError(
+ "Wrong patch spec '%s', when using a dict 'name' entry must be supplied", str(patch_spec),
+ exit_code=EasyBuildExit.EASYCONFIG_ERROR
+ )
if 'copy' not in patch_info.keys():
validate_patch_spec(patch_info['name'])
else:
@@ -1509,9 +1567,11 @@ def create_patch_info(patch_spec):
"this implies you want to copy a file to the 'copy' location)",
str(patch_spec))
else:
- error_msg = "Wrong patch spec, should be string, 2-tuple with patch name + argument, or a dict " \
- "(with possible keys %s): %s" % (valid_keys, patch_spec)
- raise EasyBuildError(error_msg)
+ error_msg = (
+ "Wrong patch spec, should be string, 2-tuple with patch name + argument, or a dict "
+ f"(with possible keys {valid_keys}): {patch_spec}"
+ )
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.EASYCONFIG_ERROR)
return patch_info
@@ -1519,8 +1579,10 @@ def create_patch_info(patch_spec):
def validate_patch_spec(patch_spec):
allowed_patch_exts = ['.patch' + x for x in ('',) + ZIPPED_PATCH_EXTS]
if not any(patch_spec.endswith(x) for x in allowed_patch_exts):
- raise EasyBuildError("Wrong patch spec (%s), extension type should be any of %s." %
- (patch_spec, ', '.join(allowed_patch_exts)))
+ raise EasyBuildError(
+ "Wrong patch spec (%s), extension type should be any of %s.", patch_spec, ', '.join(allowed_patch_exts),
+ exit_code=EasyBuildExit.EASYCONFIG_ERROR
+ )
def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False):
@@ -1611,7 +1673,7 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False
if res.exit_code:
msg = f"Couldn't apply patch file {patch_file}. "
msg += f"Process exited with code {res.exit_code}: {res.output}"
- raise EasyBuildError(msg)
+ raise EasyBuildError(msg, exit_code=EasyBuildExit.FAIL_PATCH_APPLY)
return True
@@ -2352,20 +2414,58 @@ def copy_file(path, target_path, force_in_dry_run=False):
try:
# check whether path to copy exists (we could be copying a broken symlink, which is supported)
path_exists = os.path.exists(path)
+ # If target is a folder, the target_path will be a file with the same name inside the folder
+ if os.path.isdir(target_path):
+ target_path = os.path.join(target_path, os.path.basename(path))
target_exists = os.path.exists(target_path)
+
if target_exists and path_exists and os.path.samefile(path, target_path):
_log.debug("Not copying %s to %s since files are identical", path, target_path)
# if target file exists and is owned by someone else than the current user,
- # try using shutil.copyfile to just copy the file contents
- # since shutil.copy2 will fail when trying to copy over file metadata (since chown requires file ownership)
+ # copy just the file contents (shutil.copyfile instead of shutil.copy2)
+ # since copying the file metadata/permissions will fail since chown requires file ownership
elif target_exists and os.stat(target_path).st_uid != os.getuid():
shutil.copyfile(path, target_path)
_log.info("Copied contents of file %s to %s", path, target_path)
else:
mkdir(os.path.dirname(target_path), parents=True)
if path_exists:
- shutil.copy2(path, target_path)
- _log.info("%s copied to %s", path, target_path)
+ try:
+ # on filesystems that support extended file attributes, copying read-only files with
+ # shutil.copy2() will give a PermissionError, when using Python < 3.7
+ # see https://bugs.python.org/issue24538
+ shutil.copy2(path, target_path)
+ _log.info("%s copied to %s", path, target_path)
+ # catch the more general OSError instead of PermissionError,
+ # since Python 2.7 doesn't support PermissionError
+ except OSError as err:
+ # if file is writable (not read-only), then we give up since it's not a simple permission error
+ if os.path.exists(target_path) and os.stat(target_path).st_mode & stat.S_IWUSR:
+ raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+
+ pyver = LooseVersion(platform.python_version())
+ if pyver >= LooseVersion('3.7'):
+ raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+ elif LooseVersion('3.7') > pyver >= LooseVersion('3'):
+ if not isinstance(err, PermissionError):
+ raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+
+ # double-check whether the copy actually succeeded
+ if not os.path.exists(target_path) or not filecmp.cmp(path, target_path, shallow=False):
+ raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+
+ try:
+ # re-enable user write permissions in target, copy xattrs, then remove write perms again
+ adjust_permissions(target_path, stat.S_IWUSR)
+ shutil._copyxattr(path, target_path)
+ adjust_permissions(target_path, stat.S_IWUSR, add=False)
+ except OSError as err:
+ raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+
+ msg = ("Failed to copy extended attributes from file %s to %s, due to a bug in shutil (see "
+ "https://bugs.python.org/issue24538). Copy successful with workaround.")
+ _log.info(msg, path, target_path)
+
elif os.path.islink(path):
if os.path.isdir(target_path):
target_path = os.path.join(target_path, os.path.basename(path))
@@ -2540,12 +2640,12 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs):
raise EasyBuildError("Specified path to copy is not an existing file or directory: %s", path)
-def get_source_tarball_from_git(filename, targetdir, git_config):
+def get_source_tarball_from_git(filename, target_dir, git_config):
"""
Downloads a git repository, at a specific tag or commit, recursively or not, and make an archive with it
:param filename: name of the archive to save the code to (must be .tar.gz)
- :param targetdir: target directory where to save the archive to
+ :param target_dir: target directory where to save the archive to
:param git_config: dictionary containing url, repo_name, recursive, and one of tag or commit
"""
# sanity check on git_config value being passed
@@ -2584,8 +2684,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
raise EasyBuildError("git_config currently only supports filename ending in .tar.gz")
# prepare target directory and clone repository
- mkdir(targetdir, parents=True)
- targetpath = os.path.join(targetdir, filename)
+ mkdir(target_dir, parents=True)
# compose 'git clone' command, and run it
if extra_config_params:
@@ -2643,7 +2742,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
work_dir = os.path.join(tmpdir, repo_name) if repo_name else tmpdir
res = run_shell_cmd(cmd, fail_on_error=False, work_dir=work_dir, hidden=True, verbose_dry_run=True)
- if res.exit_code != 0 or tag not in res.output.splitlines():
+ if res.exit_code != EasyBuildExit.SUCCESS or tag not in res.output.splitlines():
msg = f"Tag {tag} was not downloaded in the first try due to {url}/{repo_name} containing a branch"
msg += f" with the same name. You might want to alert the maintainers of {repo_name} about that issue."
print_warning(msg)
@@ -2668,17 +2767,36 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
for cmd in cmds:
run_shell_cmd(cmd, work_dir=work_dir, hidden=True, verbose_dry_run=True)
- # create an archive and delete the git repo directory
+ # Create archive
+ archive_path = os.path.join(target_dir, filename)
+
if keep_git_dir:
- tar_cmd = ['tar', 'cfvz', targetpath, repo_name]
+ # create archive of git repo including .git directory
+ tar_cmd = ['tar', 'cfvz', archive_path, repo_name]
else:
- tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name]
+ # create reproducible archive
+ # see https://reproducible-builds.org/docs/archives/
+ tar_cmd = [
+ # print names of all files and folders excluding .git directory
+ 'find', repo_name, '-name ".git"', '-prune', '-o', '-print0',
+ # reset access and modification timestamps to epoch 0 (equivalent to --mtime in GNU tar)
+ '-exec', 'touch', '--date=@0', '{}', r'\;',
+ # reset file permissions of cloned repo (equivalent to --mode in GNU tar)
+ '-exec', 'chmod', '"go+u,go-w"', '{}', r'\;', '|',
+ # sort file list (equivalent to --sort in GNU tar)
+ 'LC_ALL=C', 'sort', '--zero-terminated', '|',
+ # create tarball in GNU format with ownership and permissions reset
+ 'tar', '--create', '--no-recursion', '--owner=0', '--group=0', '--numeric-owner',
+ '--format=gnu', '--null', '--files-from', '-', '|',
+ # compress tarball with gzip without original file name and timestamp
+ 'gzip', '--no-name', '>', archive_path
+ ]
run_shell_cmd(' '.join(tar_cmd), work_dir=tmpdir, hidden=True, verbose_dry_run=True)
# cleanup (repo_name dir does not exist in dry run mode)
remove(tmpdir)
- return targetpath
+ return archive_path
def move_file(path, target_path, force_in_dry_run=False):
diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py
index 875724acb4..a4ad1bc479 100644
--- a/easybuild/tools/github.py
+++ b/easybuild/tools/github.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -54,13 +54,14 @@
from easybuild.framework.easyconfig.easyconfig import process_easyconfig
from easybuild.framework.easyconfig.parser import EasyConfigParser
from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg, print_warning
from easybuild.tools.config import build_option
-from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_framework_files
+from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_file, copy_framework_files
from easybuild.tools.filetools import det_patched_files, download_file, extract_file
from easybuild.tools.filetools import get_easyblock_class_name, mkdir, read_file, symlink, which, write_file
from easybuild.tools.systemtools import UNKNOWN, get_tool_version
from easybuild.tools.utilities import nub, only_if_module_is_available
+from easybuild.tools.version import FRAMEWORK_VERSION, different_major_versions
_log = fancylogger.getLogger('github', fname=False)
@@ -207,7 +208,7 @@ def listdir(self, path):
return listing[1]
else:
self.log.warning("error: %s" % str(listing))
- raise EasyBuildError("Invalid response from github (I/O error)")
+ raise EasyBuildError("Invalid response from github (I/O error)", exit_code=EasyBuildExit.FAIL_GITHUB)
def walk(self, top=None, topdown=True):
"""
@@ -314,9 +315,12 @@ def github_api_put_request(request_f, github_user=None, token=None, **kwargs):
if status == 200:
_log.info("Put request successful: %s", data['message'])
elif status in [405, 409]:
- raise EasyBuildError("FAILED: %s", data['message'])
+ raise EasyBuildError("FAILED: %s", data['message'], exit_code=EasyBuildExit.FAIL_GITHUB)
else:
- raise EasyBuildError("FAILED: %s", data.get('message', "(unknown reason)"))
+ raise EasyBuildError(
+ "FAILED: %s", data.get('message', "(unknown reason)"),
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
_log.debug("get request result for %s: status: %d, data: %s", url.url, status, data)
return (status, data)
@@ -338,34 +342,39 @@ def fetch_latest_commit_sha(repo, account, branch=None, github_user=None, token=
status, data = github_api_get_request(lambda x: x.repos[account][repo].branches,
github_user=github_user, token=token, per_page=GITHUB_MAX_PER_PAGE)
if status != HTTP_STATUS_OK:
- raise EasyBuildError("Failed to get latest commit sha for branch %s from %s/%s (status: %d %s)",
- branch, account, repo, status, data)
+ raise EasyBuildError(
+ "Failed to get latest commit sha for branch %s from %s/%s (status: %d %s)",
+ branch, account, repo, status, data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
res = None
for entry in data:
- if entry[u'name'] == branch:
+ if entry['name'] == branch:
res = entry['commit']['sha']
break
if res is None:
- error_msg = "No branch with name %s found in repo %s/%s" % (branch, account, repo)
+ error_msg = f"No branch with name {branch} found in repo {account}/{repo}"
if len(data) >= GITHUB_MAX_PER_PAGE:
- error_msg += "; only %d branches were checked (too many branches in %s/%s?)" % (len(data), account, repo)
- raise EasyBuildError(error_msg + ': ' + ', '.join([x[u'name'] for x in data]))
+ error_msg += f"; only {len(data)} branches were checked (too many branches in {account}/{repo}?)"
+ error_msg += ": " + ", ".join([x['name'] for x in data])
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.FAIL_GITHUB)
return res
-def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_MAIN, path=None, github_user=None):
+def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, account=GITHUB_EB_MAIN, path=None,
+ github_user=None):
"""
Download entire GitHub repo as a tar.gz archive, and extract it into specified path.
:param repo: repo to download
:param branch: branch to download
+ :param commit: commit to download
:param account: GitHub account to download repo from
:param path: path to extract to
:param github_user: name of GitHub user to use
"""
- if branch is None:
+ if branch is None and commit is None:
branch = pick_default_branch(account)
# make sure path exists, create it if necessary
@@ -376,9 +385,27 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_M
path = os.path.join(path, account)
mkdir(path, parents=True)
- extracted_dir_name = '%s-%s' % (repo, branch)
- base_name = '%s.tar.gz' % branch
- latest_commit_sha = fetch_latest_commit_sha(repo, account, branch, github_user=github_user)
+ if commit:
+ # make sure that full commit SHA is provided
+ commit_sha1_regex = re.compile('[0-9a-f]{40}')
+ if commit_sha1_regex.match(commit):
+ _log.info("Valid commit SHA provided for downloading %s/%s: %s", account, repo, commit)
+ else:
+ error_msg = r"Specified commit SHA %s for downloading %s/%s is not valid, "
+ error_msg += "must be full SHA-1 (40 chars)"
+ raise EasyBuildError(error_msg, commit, account, repo, exit_code=EasyBuildExit.VALUE_ERROR)
+
+ extracted_dir_name = '%s-%s' % (repo, commit)
+ base_name = '%s.tar.gz' % commit
+ latest_commit_sha = commit
+ elif branch:
+ extracted_dir_name = '%s-%s' % (repo, branch)
+ base_name = '%s.tar.gz' % branch
+ latest_commit_sha = fetch_latest_commit_sha(repo, account, branch, github_user=github_user)
+ else:
+ raise EasyBuildError(
+ "Either branch or commit should be specified in download_repo", exit_code=EasyBuildExit.VALUE_ERROR
+ )
expected_path = os.path.join(path, extracted_dir_name)
latest_sha_path = os.path.join(expected_path, 'latest-sha')
@@ -394,19 +421,36 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_M
target_path = os.path.join(path, base_name)
_log.debug("downloading repo %s/%s as archive from %s to %s" % (account, repo, url, target_path))
- download_file(base_name, url, target_path, forced=True, trace=False)
- _log.debug("%s downloaded to %s, extracting now" % (base_name, path))
+ downloaded_path = download_file(base_name, url, target_path, forced=True, trace=False)
+ if downloaded_path is None:
+ raise EasyBuildError(
+ "Failed to download tarball for %s/%s commit %s", account, repo, commit,
+ exit_code=EasyBuildExit.FAIL_DOWNLOAD
+ )
+ else:
+ _log.debug("%s downloaded to %s, extracting now", base_name, path)
base_dir = extract_file(target_path, path, forced=True, change_into_dir=False, trace=False)
extracted_path = os.path.join(base_dir, extracted_dir_name)
# check if extracted_path exists
if not os.path.isdir(extracted_path):
- raise EasyBuildError("%s should exist and contain the repo %s at branch %s", extracted_path, repo, branch)
+ error_msg = "%s should exist and contain the repo %s " % (extracted_path, repo)
+ if branch:
+ error_msg += "at branch " + branch
+ elif commit:
+ error_msg += "at commit " + commit
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.FAIL_EXTRACT)
write_file(latest_sha_path, latest_commit_sha, forced=True)
- _log.debug("Repo %s at branch %s extracted into %s" % (repo, branch, extracted_path))
+ log_msg = "Repo %s at %%s extracted into %s" % (repo, extracted_path)
+ if branch:
+ log_msg = log_msg % ('branch ' + branch)
+ elif commit:
+ log_msg = log_msg % ('commit ' + commit)
+ _log.debug(log_msg)
+
return extracted_path
@@ -451,19 +495,22 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
if path is None:
if github_repo == GITHUB_EASYCONFIGS_REPO:
- pr_paths = build_option('pr_paths')
- if pr_paths:
+ extra_ec_paths = build_option('extra_ec_paths')
+ if extra_ec_paths:
# figure out directory for this specific PR (see also alt_easyconfig_paths)
- cands = [p for p in pr_paths if p.endswith('files_pr%s' % pr)]
+ cands = [p for p in extra_ec_paths if p.endswith('files_pr%s' % pr)]
if len(cands) == 1:
path = cands[0]
else:
- raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", pr, pr_paths)
+ raise EasyBuildError(
+ "Failed to isolate path for PR #%s from list of PR paths: %s", pr, extra_ec_paths,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
elif github_repo == GITHUB_EASYBLOCKS_REPO:
path = os.path.join(tempfile.gettempdir(), 'ebs_pr%s' % pr)
else:
- raise EasyBuildError("Unknown repo: %s" % github_repo)
+ raise EasyBuildError("Unknown repo: %s", github_repo, exit_code=EasyBuildExit.OPTION_ERROR)
if path is None:
path = tempfile.mkdtemp()
@@ -479,7 +526,9 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
elif github_repo == GITHUB_EASYBLOCKS_REPO:
easyfiles = 'easyblocks'
else:
- raise EasyBuildError("Don't know how to fetch files from repo %s", github_repo)
+ raise EasyBuildError(
+ "Don't know how to fetch files from repo %s", github_repo, exit_code=EasyBuildExit.OPTION_ERROR
+ )
subdir = os.path.join('easybuild', easyfiles)
@@ -553,13 +602,51 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
if os.path.exists(full_path):
files.append(full_path)
else:
- raise EasyBuildError("Couldn't find path to patched file %s", full_path)
+ raise EasyBuildError(
+ "Couldn't find path to patched file %s", full_path, exit_code=EasyBuildExit.OPTION_ERROR
+ )
+
+ if github_repo == GITHUB_EASYCONFIGS_REPO:
+ ver_file = os.path.join(final_path, 'setup.py')
+ elif github_repo == GITHUB_EASYBLOCKS_REPO:
+ ver_file = os.path.join(final_path, 'easybuild', 'easyblocks', '__init__.py')
+ else:
+ raise EasyBuildError("Don't know how to determine version for repo %s", github_repo)
+
+ # take into account that the file we need to determine repo version may not be available,
+ # for example when a closed PR is used (since then we only download files patched by the PR)
+ if os.path.exists(ver_file):
+ ver = _get_version_for_repo(ver_file)
+ if different_major_versions(FRAMEWORK_VERSION, ver):
+ raise EasyBuildError("Framework (%s) is a different major version than used in %s/%s PR #%s (%s)",
+ FRAMEWORK_VERSION, github_account, github_repo, pr, ver)
return files
+def _get_version_for_repo(filename):
+ """Extract version from filename."""
+ _log.debug("Extract version from %s" % filename)
+
+ try:
+ ver_line = ""
+ with open(filename) as f:
+ for line in f.readlines():
+ if line.startswith("VERSION "):
+ ver_line = line
+ break
+
+ # version can be a string or LooseVersion
+ res = re.search(r"""^VERSION = .*['"](.*)['"].?$""", ver_line)
+
+ _log.debug("PR target version is %s" % res.group(1))
+ return res.group(1)
+ except Exception:
+ raise EasyBuildError("Couldn't determine version of PR from %s" % filename)
+
+
def fetch_easyblocks_from_pr(pr, path=None, github_user=None):
- """Fetch patched easyconfig files for a particular PR."""
+ """Fetch patched easyblocks for a particular PR."""
return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO)
@@ -568,6 +655,120 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None):
return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYCONFIGS_REPO)
+def fetch_files_from_commit(commit, files=None, path=None, github_account=None, github_repo=None):
+ """
+ Fetch files from a specific commit.
+
+ If 'files' is None, all files touched in the commit are used.
+ """
+ if github_account is None:
+ github_account = build_option('pr_target_account')
+
+ if github_repo is None:
+ github_repo = GITHUB_EASYCONFIGS_REPO
+
+ if github_repo == GITHUB_EASYCONFIGS_REPO:
+ easybuild_subdir = os.path.join('easybuild', 'easyconfigs')
+ elif github_repo == GITHUB_EASYBLOCKS_REPO:
+ easybuild_subdir = os.path.join('easybuild', 'easyblocks')
+ else:
+ raise EasyBuildError("Unknown repo: %s", github_repo)
+
+ if path is None:
+ if github_repo == GITHUB_EASYCONFIGS_REPO:
+ extra_ec_paths = build_option('extra_ec_paths')
+ if extra_ec_paths:
+ # figure out directory for this specific commit (see also alt_easyconfig_paths)
+ cands = [p for p in extra_ec_paths if p.endswith('files_commit_' + commit)]
+ if len(cands) == 1:
+ path = cands[0]
+ else:
+ raise EasyBuildError(
+ "Failed to isolate path for commit %s from list of commit paths: %s",
+ commit, extra_ec_paths, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
+ else:
+ path = os.path.join(tempfile.gettempdir(), 'ecs_commit_' + commit)
+
+ elif github_repo == GITHUB_EASYBLOCKS_REPO:
+ path = os.path.join(tempfile.gettempdir(), 'ebs_commit_' + commit)
+ else:
+ raise EasyBuildError("Unknown repo: %s", github_repo, exit_code=EasyBuildExit.OPTION_ERROR)
+
+ # if no files are specified, determine which files are touched in commit
+ if not files:
+ diff_url = os.path.join(GITHUB_URL, github_account, github_repo, 'commit', commit + '.diff')
+ diff_fn = os.path.basename(diff_url)
+ diff_filepath = os.path.join(path, diff_fn)
+ if download_file(diff_fn, diff_url, diff_filepath, forced=True, trace=False):
+ diff_txt = read_file(diff_filepath)
+ _log.debug("Diff for commit %s:\n%s", commit, diff_txt)
+
+ files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True)
+ _log.debug("List of patched files for commit %s: %s", commit, files)
+ else:
+ raise EasyBuildError(
+ "Failed to download diff for commit %s of %s/%s", commit, github_account, github_repo,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
+
+ # download tarball for specific commit
+ repo_commit = download_repo(repo=github_repo, commit=commit, account=github_account)
+
+ if github_repo == GITHUB_EASYCONFIGS_REPO:
+ files_subdir = 'easybuild/easyconfigs/'
+ elif github_repo == GITHUB_EASYBLOCKS_REPO:
+ files_subdir = 'easybuild/easyblocks/'
+ else:
+ raise EasyBuildError("Unknown repo: %s", github_repo, exit_code=EasyBuildExit.OPTION_ERROR)
+
+ # symlink subdirectories of 'easybuild/easy{blocks,configs}' into path that gets added to robot search path
+ mkdir(path, parents=True)
+ dirpath = os.path.join(repo_commit, easybuild_subdir)
+ for subdir in os.listdir(dirpath):
+ symlink(os.path.join(dirpath, subdir), os.path.join(path, subdir))
+
+ # copy specified files to directory where they're expected to be found
+ file_paths = []
+ for file in files:
+
+ # if only filename is specified, we need to determine the file path
+ if file == os.path.basename(file):
+ src_path = None
+ for (dirpath, _, filenames) in os.walk(repo_commit, topdown=True):
+ if file in filenames:
+ src_path = os.path.join(dirpath, file)
+ break
+ else:
+ src_path = os.path.join(repo_commit, file)
+
+ # strip of leading subdirectory like easybuild/easyconfigs/ or easybuild/easyblocks/
+ # because that's what expected by robot_find_easyconfig
+ if file.startswith(files_subdir):
+ file = file[len(files_subdir):]
+
+ # if file is found, copy it to dedicated directory;
+ # if not, just skip it (may be an easyconfig file in local directory);
+ if src_path and os.path.exists(src_path):
+ target_path = os.path.join(path, file)
+ copy_file(src_path, target_path)
+ file_paths.append(target_path)
+ else:
+ _log.info("File %s not found in %s, so ignoring it...", file, repo_commit)
+
+ return file_paths
+
+
+def fetch_easyblocks_from_commit(commit, files=None, path=None):
+ """Fetch easyblocks from a specified commit."""
+ return fetch_files_from_commit(commit, files=files, path=path, github_repo=GITHUB_EASYBLOCKS_REPO)
+
+
+def fetch_easyconfigs_from_commit(commit, files=None, path=None):
+ """Fetch specified easyconfig files from a specific commit."""
+ return fetch_files_from_commit(commit, files=files, path=path, github_repo=GITHUB_EASYCONFIGS_REPO)
+
+
def create_gist(txt, fn, descr=None, github_user=None, github_token=None):
"""Create a gist with the provided text."""
@@ -598,7 +799,9 @@ def create_gist(txt, fn, descr=None, github_user=None, github_token=None):
status, data = g.gists.post(body=body)
if status != HTTP_STATUS_CREATED:
- raise EasyBuildError("Failed to create gist; status %s, data: %s", status, data)
+ raise EasyBuildError(
+ "Failed to create gist; status %s, data: %s", status, data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
return data['html_url']
@@ -613,7 +816,9 @@ def delete_gist(gist_id, github_user=None, github_token=None):
status, data = gh.gists[gist_id].delete()
if status != HTTP_STATUS_NO_CONTENT:
- raise EasyBuildError("Failed to delete gist with ID %s: status %s, data: %s", status, data)
+ raise EasyBuildError(
+ "Failed to delete gist with ID %s: status %s, data: %s", status, data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
def post_comment_in_issue(issue, txt, account=GITHUB_EB_MAIN, repo=GITHUB_EASYCONFIGS_REPO, github_user=None):
@@ -622,7 +827,10 @@ def post_comment_in_issue(issue, txt, account=GITHUB_EB_MAIN, repo=GITHUB_EASYCO
try:
issue = int(issue)
except ValueError as err:
- raise EasyBuildError("Failed to parse specified pull request number '%s' as an int: %s; ", issue, err)
+ raise EasyBuildError(
+ "Failed to parse specified pull request number '%s' as an int: %s; ", issue, err,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
dry_run = build_option('dry_run') or build_option('extended_dry_run')
@@ -639,7 +847,10 @@ def post_comment_in_issue(issue, txt, account=GITHUB_EB_MAIN, repo=GITHUB_EASYCO
status, data = pr_url.comments.post(body={'body': txt})
if not status == HTTP_STATUS_CREATED:
- raise EasyBuildError("Failed to create comment in PR %s#%d; status %s, data: %s", repo, issue, status, data)
+ raise EasyBuildError(
+ "Failed to create comment in PR %s#%d; status %s, data: %s", repo, issue, status, data,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
def init_repo(path, repo_name, silent=False):
@@ -665,13 +876,15 @@ def init_repo(path, repo_name, silent=False):
workrepo = git.Repo(workdir)
workrepo.clone(repo_path)
except GitCommandError as err:
- raise EasyBuildError("Failed to clone git repo at %s: %s", workdir, err)
+ raise EasyBuildError(
+ "Failed to clone git repo at %s: %s", workdir, err, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# initalize repo in repo_path
try:
repo = git.Repo.init(repo_path)
except GitCommandError as err:
- raise EasyBuildError("Failed to init git repo at %s: %s", repo_path, err)
+ raise EasyBuildError("Failed to init git repo at %s: %s", repo_path, err, exit_code=EasyBuildExit.FAIL_GITHUB)
_log.debug("temporary git working directory ready at %s", repo_path)
@@ -691,7 +904,7 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa
_log.debug("Cloning from %s", github_url)
if target_account is None:
- raise EasyBuildError("target_account not specified in setup_repo_from!")
+ raise EasyBuildError("target_account not specified in setup_repo_from!", exit_code=EasyBuildExit.OPTION_ERROR)
# salt to use for names of remotes/branches that are created
salt = ''.join(random.choice(ascii_letters) for _ in range(5))
@@ -700,7 +913,7 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa
origin = git_repo.create_remote(remote_name, github_url)
if not origin.exists():
- raise EasyBuildError("%s does not exist?", github_url)
+ raise EasyBuildError("%s does not exist?", github_url, exit_code=EasyBuildExit.FAIL_GITHUB)
# git fetch
# can't use --depth to only fetch a shallow copy, since pushing to another repo from a shallow copy doesn't work
@@ -709,21 +922,32 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa
try:
res = origin.fetch()
except GitCommandError as err:
- raise EasyBuildError("Failed to fetch branch '%s' from %s: %s", branch_name, github_url, err)
+ raise EasyBuildError(
+ "Failed to fetch branch '%s' from %s: %s", branch_name, github_url, err,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
if res:
if res[0].flags & res[0].ERROR:
- raise EasyBuildError("Fetching branch '%s' from remote %s failed: %s", branch_name, origin, res[0].note)
+ raise EasyBuildError(
+ "Fetching branch '%s' from remote %s failed: %s", branch_name, origin, res[0].note,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
else:
_log.debug("Fetched branch '%s' from remote %s (note: %s)", branch_name, origin, res[0].note)
else:
- raise EasyBuildError("Fetching branch '%s' from remote %s failed: empty result", branch_name, origin)
+ raise EasyBuildError(
+ "Fetching branch '%s' from remote %s failed: empty result", branch_name, origin,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# git checkout -b ; git pull
try:
origin_branch = getattr(origin.refs, branch_name)
except AttributeError:
- raise EasyBuildError("Branch '%s' not found at %s", branch_name, github_url)
+ raise EasyBuildError(
+ "Branch '%s' not found at %s", branch_name, github_url, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
_log.debug("Checking out branch '%s' from remote %s", branch_name, github_url)
try:
@@ -734,7 +958,10 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa
try:
origin_branch.checkout(b=alt_branch, force=True)
except GitCommandError as err:
- raise EasyBuildError("Failed to check out branch '%s' from repo at %s: %s", alt_branch, github_url, err)
+ raise EasyBuildError(
+ "Failed to check out branch '%s' from repo at %s: %s", alt_branch, github_url, err,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
return remote_name
@@ -770,7 +997,7 @@ def setup_repo(git_repo, target_account, target_repo, branch_name, silent=False,
if res:
return res
else:
- raise EasyBuildError('\n'.join(errors))
+ raise EasyBuildError('\n'.join(errors), exit_code=EasyBuildExit.FAIL_GITHUB)
@only_if_module_is_available('git', pkgname='GitPython')
@@ -802,14 +1029,20 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
ec_paths.append(path)
if non_existing_paths:
- raise EasyBuildError("One or more non-existing paths specified: %s", ', '.join(non_existing_paths))
+ raise EasyBuildError(
+ "One or more non-existing paths specified: %s", ', '.join(non_existing_paths),
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
if not any(paths.values()):
- raise EasyBuildError("No paths specified")
+ raise EasyBuildError("No paths specified", exit_code=EasyBuildExit.OPTION_ERROR)
pr_target_repo = det_pr_target_repo(paths)
if pr_target_repo is None:
- raise EasyBuildError("Failed to determine target repository, please specify it via --pr-target-repo!")
+ raise EasyBuildError(
+ "Failed to determine target repository, please specify it via --pr-target-repo!",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
# initialize repository
git_working_dir = tempfile.mkdtemp(prefix='git-working-dir')
@@ -817,7 +1050,10 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
repo_path = os.path.join(git_working_dir, pr_target_repo)
if pr_target_repo not in [GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_FRAMEWORK_REPO]:
- raise EasyBuildError("Don't know how to create/update a pull request to the %s repository", pr_target_repo)
+ raise EasyBuildError(
+ "Don't know how to create/update a pull request to the %s repository", pr_target_repo,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
if start_account is None:
start_account = build_option('pr_target_account')
@@ -828,7 +1064,9 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
target_account = build_option('github_org') or build_option('github_user')
if target_account is None:
- raise EasyBuildError("--github-org or --github-user must be specified!")
+ raise EasyBuildError(
+ "--github-org or --github-user must be specified!", exit_code=EasyBuildExit.OPTION_ERROR
+ )
# if branch to start from is specified, we're updating an existing PR
start_branch = build_option('pr_target_branch')
@@ -859,8 +1097,16 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
elif pr_target_repo == GITHUB_EASYBLOCKS_REPO and all(file_info['new']):
commit_msg = "adding easyblocks: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo'])
else:
+ msg = ''
+ modified_files = [os.path.basename(p) for new, p in zip(file_info['new'], file_info['paths_in_repo'])
+ if not new]
+ if modified_files:
+ msg += '\nModified: ' + ', '.join(modified_files)
+ if paths['files_to_delete']:
+ msg += '\nDeleted: ' + ', '.join(paths['files_to_delete'])
raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when "
- "modifying/deleting files or targeting the framework repo.")
+ "modifying/deleting files or targeting the framework repo." + msg,
+ exit_code=EasyBuildExit.OPTION_ERROR)
# figure out to which software name patches relate, and copy them to the right place
if paths['patch_files']:
@@ -881,7 +1127,10 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
if len(hits) == 1:
deleted_paths.append(hits[0])
else:
- raise EasyBuildError("Path doesn't exist or file to delete isn't found in target branch: %s", fn)
+ raise EasyBuildError(
+ "Path doesn't exist or file to delete isn't found in target branch: %s", fn,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
dep_info = {
'ecs': [],
@@ -900,8 +1149,8 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
# only consider new easyconfig files for dependencies (not updated ones)
for idx in range(len(all_dep_info['ecs'])):
if all_dep_info['new'][idx]:
- for key in dep_info:
- dep_info[key].append(all_dep_info[key][idx])
+ for key, info in dep_info.items():
+ info.append(all_dep_info[key][idx])
# checkout target branch
if pr_branch is None:
@@ -938,8 +1187,11 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
diff_stat = git_repo.git.diff(cached=True, stat=True)
if not diff_stat:
- raise EasyBuildError("No changed files found when comparing to current develop branch. "
- "Refused to make empty pull request.")
+ raise EasyBuildError(
+ f"No changed files found when comparing to current {start_branch} branch. "
+ "Refused to make empty pull request.",
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# commit
git_repo.index.commit(commit_msg)
@@ -970,7 +1222,9 @@ def create_remote(git_repo, account, repo, https=False):
try:
remote = git_repo.create_remote(remote_name, github_url)
except GitCommandError as err:
- raise EasyBuildError("Failed to create remote %s for %s: %s", remote_name, github_url, err)
+ raise EasyBuildError(
+ "Failed to create remote %s for %s: %s", remote_name, github_url, err, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
return remote
@@ -985,7 +1239,9 @@ def push_branch_to_github(git_repo, target_account, target_repo, branch):
:param branch: name of branch to push
"""
if target_account is None:
- raise EasyBuildError("target_account not specified in push_branch_to_github!")
+ raise EasyBuildError(
+ "target_account not specified in push_branch_to_github!", exit_code=EasyBuildExit.OPTION_ERROR
+ )
# push to GitHub
remote = create_remote(git_repo, target_account, target_repo)
@@ -1002,17 +1258,24 @@ def push_branch_to_github(git_repo, target_account, target_repo, branch):
try:
res = remote.push(branch)
except GitCommandError as err:
- raise EasyBuildError("Failed to push branch '%s' to GitHub (%s): %s", branch, github_url, err)
+ raise EasyBuildError(
+ "Failed to push branch '%s' to GitHub (%s): %s", branch, github_url, err,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
if res:
if res[0].ERROR & res[0].flags:
- raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: %s",
- branch, remote, github_url, res[0].summary)
+ raise EasyBuildError(
+ "Pushing branch '%s' to remote %s (%s) failed: %s", branch, remote, github_url, res[0].summary,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
else:
_log.debug("Pushed branch %s to remote %s (%s): %s", branch, remote, github_url, res[0].summary)
else:
- raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: empty result",
- branch, remote, github_url)
+ raise EasyBuildError(
+ "Pushing branch '%s' to remote %s (%s) failed: empty result", branch, remote, github_url,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
def is_patch_for(patch_name, ec):
@@ -1069,7 +1332,10 @@ def det_patch_specs(patch_paths, file_info, ec_dirs):
patch_specs.append((patch_path, soft_name))
else:
# still nothing found
- raise EasyBuildError("Failed to determine software name to which patch file %s relates", patch_path)
+ raise EasyBuildError(
+ "Failed to determine software name to which patch file %s relates", patch_path,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
return patch_specs
@@ -1336,7 +1602,7 @@ def close_pr(pr, motivation_msg=None):
"""
github_user = build_option('github_user')
if github_user is None:
- raise EasyBuildError("GitHub user must be specified to use --close-pr")
+ raise EasyBuildError("GitHub user must be specified to use --close-pr", exit_code=EasyBuildExit.OPTION_ERROR)
pr_target_account = build_option('pr_target_account')
pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
@@ -1344,7 +1610,10 @@ def close_pr(pr, motivation_msg=None):
pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user, full=True)
if pr_data['state'] == GITHUB_STATE_CLOSED:
- raise EasyBuildError("PR #%d from %s/%s is already closed.", pr, pr_target_account, pr_target_repo)
+ raise EasyBuildError(
+ "PR #%d from %s/%s is already closed.", pr, pr_target_account, pr_target_repo,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
pr_owner = pr_data['user']['login']
msg = "\n%s/%s PR #%s was submitted by %s, " % (pr_target_account, pr_target_repo, pr, pr_owner)
@@ -1361,8 +1630,10 @@ def close_pr(pr, motivation_msg=None):
possible_reasons = reasons_for_closing(pr_data)
if not possible_reasons:
- raise EasyBuildError("No reason specified and none found from PR data, "
- "please use --close-pr-reasons or --close-pr-msg")
+ raise EasyBuildError(
+ "No reason specified and none found from PR data, please use --close-pr-reasons or --close-pr-msg",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
else:
motivation_msg = ", ".join([VALID_CLOSE_PR_REASONS[reason] for reason in possible_reasons])
print_msg("\nNo reason specified but found possible reasons: %s.\n" % motivation_msg, prefix=False)
@@ -1381,18 +1652,26 @@ def close_pr(pr, motivation_msg=None):
else:
github_token = fetch_github_token(github_user)
if github_token is None:
- raise EasyBuildError("GitHub token for user '%s' must be available to use --close-pr", github_user)
+ raise EasyBuildError(
+ "GitHub token for user '%s' must be available to use --close-pr", github_user,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
g = RestClient(GITHUB_API_URL, username=github_user, token=github_token)
pull_url = g.repos[pr_target_account][pr_target_repo].pulls[pr]
body = {'state': 'closed'}
status, data = pull_url.post(body=body)
if not status == HTTP_STATUS_OK:
- raise EasyBuildError("Failed to close PR #%s; status %s, data: %s", pr, status, data)
+ raise EasyBuildError(
+ "Failed to close PR #%s; status %s, data: %s", pr, status, data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
if reopen:
body = {'state': 'open'}
status, data = pull_url.post(body=body)
if not status == HTTP_STATUS_OK:
- raise EasyBuildError("Failed to reopen PR #%s; status %s, data: %s", pr, status, data)
+ raise EasyBuildError(
+ "Failed to reopen PR #%s; status %s, data: %s", pr, status, data,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
def list_prs(params, per_page=GITHUB_MAX_PER_PAGE, github_user=None):
@@ -1428,7 +1707,7 @@ def merge_pr(pr):
"""
github_user = build_option('github_user')
if github_user is None:
- raise EasyBuildError("GitHub user must be specified to use --merge-pr")
+ raise EasyBuildError("GitHub user must be specified to use --merge-pr", exit_code=EasyBuildExit.OPTION_ERROR)
pr_target_account = build_option('pr_target_account')
pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
@@ -1440,16 +1719,16 @@ def merge_pr(pr):
msg += "\nPR title: %s\n\n" % pr_data['title']
print_msg(msg, prefix=False)
if pr_data['user']['login'] == github_user:
- raise EasyBuildError("Please do not merge your own PRs!")
+ raise EasyBuildError("Please do not merge your own PRs!", exit_code=EasyBuildExit.OPTION_ERROR)
force = build_option('force')
dry_run = build_option('dry_run') or build_option('extended_dry_run')
if not dry_run:
if pr_data['merged']:
- raise EasyBuildError("This PR is already merged.")
+ raise EasyBuildError("This PR is already merged.", exit_code=EasyBuildExit.OPTION_ERROR)
elif pr_data['state'] == GITHUB_STATE_CLOSED:
- raise EasyBuildError("This PR is closed.")
+ raise EasyBuildError("This PR is closed.", exit_code=EasyBuildExit.OPTION_ERROR)
def merge_url(gh):
"""Utility function to fetch merge URL for a specific PR."""
@@ -1514,7 +1793,7 @@ def post_pr_labels(pr, labels):
pr_url = g.repos[pr_target_account][pr_target_repo].issues[pr]
try:
- status, data = pr_url.labels.post(body=labels)
+ status, _ = pr_url.labels.post(body=labels)
if status == HTTP_STATUS_OK:
print_msg("Added labels %s to PR#%s" % (', '.join(labels), pr), log=_log, prefix=False)
return True
@@ -1533,7 +1812,10 @@ def add_pr_labels(pr, branch=GITHUB_DEVELOP_BRANCH):
"""
pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
if pr_target_repo != GITHUB_EASYCONFIGS_REPO:
- raise EasyBuildError("Adding labels to PRs for repositories other than easyconfigs hasn't been implemented yet")
+ raise EasyBuildError(
+ "Adding labels to PRs for repositories other than easyconfigs hasn't been implemented yet",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
tmpdir = tempfile.mkdtemp()
@@ -1641,11 +1923,17 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None,
# fetch GitHub token (required to perform actions on GitHub)
github_user = build_option('github_user')
if github_user is None:
- raise EasyBuildError("GitHub user must be specified to open a pull request")
+ raise EasyBuildError(
+ "GitHub user must be specified to open a pull request",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
github_token = fetch_github_token(github_user)
if github_token is None:
- raise EasyBuildError("GitHub token for user '%s' must be available to open a pull request", github_user)
+ raise EasyBuildError(
+ "GitHub token for user '%s' must be available to open a pull request", github_user,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# GitHub organisation or GitHub user where branch is located
github_account = build_option('github_org') or github_user
@@ -1707,7 +1995,10 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None,
print_msg('\n'.join(msg), log=_log)
else:
- raise EasyBuildError("No changes in '%s' branch compared to current 'develop' branch!", branch_name)
+ raise EasyBuildError(
+ f"No changes in '{branch_name}' branch compared to current '{pr_target_branch}' branch!",
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# copy repo while target branch is still checked out
tmpdir = tempfile.mkdtemp()
@@ -1740,8 +2031,10 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None,
title = "new easyblock%s for %s" % (plural, (', '.join(file_info['eb_names'])))
if title is None:
- raise EasyBuildError("Don't know how to make a PR title for this PR. "
- "Please include a title (use --pr-title)")
+ raise EasyBuildError(
+ "Don't know how to make a PR title for this PR. Please include a title (use --pr-title)",
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
full_descr = "(created using `eb --new-pr`)\n"
if descr is not None:
@@ -1778,7 +2071,10 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None,
}
status, data = pulls_url.post(body=body)
if not status == HTTP_STATUS_CREATED:
- raise EasyBuildError("Failed to open PR for branch %s; status %s, data: %s", branch_name, status, data)
+ raise EasyBuildError(
+ "Failed to open PR for branch %s; status %s, data: %s", branch_name, status, data,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
print_msg("Opened pull request: %s" % data['html_url'], log=_log, prefix=False)
@@ -1813,11 +2109,11 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None):
patch = patch[0]
elif isinstance(patch, dict):
patch_info = {}
- for key in patch.keys():
- patch_info[key] = patch[key]
- if 'name' not in patch_info.keys():
- raise EasyBuildError("Wrong patch spec '%s', when using a dict 'name' entry must be supplied",
- str(patch))
+ for key, cur_patch in patch.items():
+ patch_info[key] = cur_patch
+ if 'name' not in patch_info:
+ msg = f"Wrong patch spec '{patch}', when using a dict 'name' entry must be supplied"
+ raise EasyBuildError(msg, exit_code=EasyBuildExit.EASYCONFIG_ERROR)
patch = patch_info['name']
if patch not in paths['patch_files'] and not os.path.isfile(os.path.join(os.path.dirname(ec_path),
@@ -1836,7 +2132,7 @@ def det_account_branch_for_pr(pr_id, github_user=None, pr_target_repo=None):
github_user = build_option('github_user')
if github_user is None:
- raise EasyBuildError("GitHub username (--github-user) must be specified!")
+ raise EasyBuildError("GitHub username (--github-user) must be specified!", exit_code=EasyBuildExit.OPTION_ERROR)
pr_target_account = build_option('pr_target_account')
if pr_target_repo is None:
@@ -1908,7 +2204,10 @@ def update_branch(branch_name, paths, ecs, github_account=None, commit_msg=None)
commit_msg = build_option('pr_commit_msg')
if commit_msg is None:
- raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when using --update-pr")
+ raise EasyBuildError(
+ "A meaningful commit message must be specified via --pr-commit-msg when using --update-pr",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
if github_account is None:
github_account = build_option('github_user') or build_option('github_org')
@@ -1939,7 +2238,10 @@ def update_pr(pr_id, paths, ecs, commit_msg=None):
pr_target_repo = det_pr_target_repo(paths)
if pr_target_repo is None:
- raise EasyBuildError("Failed to determine target repository, please specify it via --pr-target-repo!")
+ raise EasyBuildError(
+ "Failed to determine target repository, please specify it via --pr-target-repo!",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
github_account, branch_name = det_account_branch_for_pr(pr_id, pr_target_repo=pr_target_repo)
@@ -2001,7 +2303,9 @@ def check_github():
print_msg("OK\n", log=_log, prefix=False)
else:
print_msg("FAIL (%s)", ', '.join(online_state), log=_log, prefix=False)
- raise EasyBuildError("checking status of GitHub integration must be done online")
+ raise EasyBuildError(
+ "checking status of GitHub integration must be done online", exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# GitHub user
print_msg("* GitHub user...", log=_log, prefix=False, newline=False)
@@ -2178,7 +2482,7 @@ def check_github():
msg = '\n'.join([
'',
"One or more checks FAILed, GitHub configuration not fully complete!",
- "See http://easybuild.readthedocs.org/en/latest/Integration_with_GitHub.html#configuration for help.",
+ "See https://docs.easybuild.io/integration-with-github/#github_configuration for help.",
'',
])
print_msg(msg, log=_log, prefix=False)
@@ -2235,7 +2539,9 @@ def install_github_token(github_user, silent=False):
:param silent: keep quiet (don't print any messages)
"""
if github_user is None:
- raise EasyBuildError("GitHub user must be specified to install GitHub token")
+ raise EasyBuildError(
+ "GitHub user must be specified to install GitHub token", exit_code=EasyBuildExit.OPTION_ERROR
+ )
# check if there's a token available already
current_token = fetch_github_token(github_user)
@@ -2246,8 +2552,10 @@ def install_github_token(github_user, silent=False):
msg = "WARNING: overwriting installed token '%s' for user '%s'..." % (current_token, github_user)
print_msg(msg, prefix=False, silent=silent)
else:
- raise EasyBuildError("Installed token '%s' found for user '%s', not overwriting it without --force",
- current_token, github_user)
+ raise EasyBuildError(
+ "Installed token '%s' found for user '%s', not overwriting it without --force",
+ current_token, github_user, exit_code=EasyBuildExit.OPTION_ERROR
+ )
# get token to install
token = getpass.getpass(prompt="Token: ").strip()
@@ -2258,7 +2566,10 @@ def install_github_token(github_user, silent=False):
if valid_token:
print_msg("Token seems to be valid, installing it.", prefix=False, silent=silent)
else:
- raise EasyBuildError("Token validation failed, not installing it. Please verify your token and try again.")
+ raise EasyBuildError(
+ "Token validation failed, not installing it. Please verify your token and try again.",
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# install token
keyring.set_password(KEYRING_GITHUB_TOKEN, github_user, token)
@@ -2332,7 +2643,7 @@ def find_easybuild_easyconfig(github_user=None):
if file_versions:
fn = sorted(file_versions)[-1][1]
else:
- raise EasyBuildError("Couldn't find any EasyBuild easyconfigs")
+ raise EasyBuildError("Couldn't find any EasyBuild easyconfigs", exit_code=EasyBuildExit.MISSING_EASYCONFIG)
eb_file = os.path.join(eb_parent_path, fn)
return eb_file
@@ -2359,8 +2670,11 @@ def check_suites_url(gh):
# first check combined commit status (set by e.g. Travis CI)
status, commit_status_data = github_api_get_request(commit_status_url, github_user)
if status != HTTP_STATUS_OK:
- raise EasyBuildError("Failed to get status of commit %s from %s/%s (status: %d %s)",
- commit_sha, account, repo, status, commit_status_data)
+ raise EasyBuildError(
+ "Failed to get status of commit %s from %s/%s (status: %d %s)",
+ commit_sha, account, repo, status, commit_status_data,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
commit_status_count = commit_status_data['total_count']
combined_commit_status = commit_status_data['state']
@@ -2408,7 +2722,9 @@ def check_suites_url(gh):
break
else:
app_name = check_suite_data.get('app', {}).get('name', 'UNKNOWN')
- raise EasyBuildError("Unknown check suite status set by %s: '%s'", app_name, status)
+ raise EasyBuildError(
+ "Unknown check suite status set by %s: '%s'", app_name, status, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
return result
@@ -2426,13 +2742,16 @@ def pr_url(gh):
try:
status, pr_data = github_api_get_request(pr_url, github_user, **parameters)
except HTTPError as err:
- raise EasyBuildError("Failed to get data for PR #%d from %s/%s (%s)\n"
- "Please check PR #, account and repo.",
- pr, pr_target_account, pr_target_repo, err)
+ raise EasyBuildError(
+ "Failed to get data for PR #%d from %s/%s (%s)\nPlease check PR #, account and repo.",
+ pr, pr_target_account, pr_target_repo, err, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
if status != HTTP_STATUS_OK:
- raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)",
- pr, pr_target_account, pr_target_repo, status, pr_data)
+ raise EasyBuildError(
+ "Failed to get data for PR #%d from %s/%s (status: %d %s)",
+ pr, pr_target_account, pr_target_repo, status, pr_data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
if full:
# also fetch status of last commit
@@ -2446,8 +2765,10 @@ def comments_url(gh):
status, comments_data = github_api_get_request(comments_url, github_user, **parameters)
if status != HTTP_STATUS_OK:
- raise EasyBuildError("Failed to get comments for PR #%d from %s/%s (status: %d %s)",
- pr, pr_target_account, pr_target_repo, status, comments_data)
+ raise EasyBuildError(
+ "Failed to get comments for PR #%d from %s/%s (status: %d %s)",
+ pr, pr_target_account, pr_target_repo, status, comments_data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
pr_data['issue_comments'] = comments_data
# also fetch reviews
@@ -2457,8 +2778,10 @@ def reviews_url(gh):
status, reviews_data = github_api_get_request(reviews_url, github_user, **parameters)
if status != HTTP_STATUS_OK:
- raise EasyBuildError("Failed to get reviews for PR #%d from %s/%s (status: %d %s)",
- pr, pr_target_account, pr_target_repo, status, reviews_data)
+ raise EasyBuildError(
+ "Failed to get reviews for PR #%d from %s/%s (status: %d %s)",
+ pr, pr_target_account, pr_target_repo, status, reviews_data, exit_code=EasyBuildExit.FAIL_GITHUB
+ )
pr_data['reviews'] = reviews_data
return pr_data, pr_url
@@ -2505,7 +2828,9 @@ def sync_pr_with_develop(pr_id):
"""Sync pull request with specified ID with current develop branch."""
github_user = build_option('github_user')
if github_user is None:
- raise EasyBuildError("GitHub user must be specified to use --sync-pr-with-develop")
+ raise EasyBuildError(
+ "GitHub user must be specified to use --sync-pr-with-develop", exit_code=EasyBuildExit.OPTION_ERROR
+ )
target_account = build_option('pr_target_account')
target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
@@ -2528,7 +2853,9 @@ def sync_branch_with_develop(branch_name):
"""Sync branch with specified name with current develop branch."""
github_user = build_option('github_user')
if github_user is None:
- raise EasyBuildError("GitHub user must be specified to use --sync-branch-with-develop")
+ raise EasyBuildError(
+ "GitHub user must be specified to use --sync-branch-with-develop", exit_code=EasyBuildExit.OPTION_ERROR
+ )
target_account = build_option('pr_target_account')
target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py
index 2bb329027b..b114224f95 100644
--- a/easybuild/tools/hooks.py
+++ b/easybuild/tools/hooks.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2017-2023 Ghent University
+# Copyright 2017-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py
index 3898facd4c..0171d74f10 100644
--- a/easybuild/tools/include.py
+++ b/easybuild/tools/include.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
# #
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/jenkins.py b/easybuild/tools/jenkins.py
index 3c00d3961c..50ee563083 100644
--- a/easybuild/tools/jenkins.py
+++ b/easybuild/tools/jenkins.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/job/backend.py b/easybuild/tools/job/backend.py
index c92ead6a58..1219883740 100644
--- a/easybuild/tools/job/backend.py
+++ b/easybuild/tools/job/backend.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -104,7 +104,7 @@ def avail_job_backends(check_usable=True):
Return all known job execution backends.
"""
import_available_modules('easybuild.tools.job')
- class_dict = dict([(x.__name__, x) for x in get_subclasses(JobBackend)])
+ class_dict = {x.__name__: x for x in get_subclasses(JobBackend)}
return class_dict
diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py
index 005cff2e90..60f7432217 100644
--- a/easybuild/tools/job/gc3pie.py
+++ b/easybuild/tools/job/gc3pie.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
# Copyright 2015 S3IT, University of Zurich
#
# This file is part of EasyBuild,
@@ -104,6 +104,10 @@ def __init__(self, *args, **kwargs):
def _check_version(self):
"""Check whether GC3Pie version complies with required version."""
+ deprecation_msg = "The GC3Pie job backend is no longer maintained and will be removed"
+ deprecation_msg += ", please use a different job backend"
+ _log.deprecated(deprecation_msg, '6.0')
+
try:
from pkg_resources import get_distribution, DistributionNotFound
pkg = get_distribution('gc3pie')
diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py
index a4a020988e..31199dd1ee 100644
--- a/easybuild/tools/job/pbs_python.py
+++ b/easybuild/tools/job/pbs_python.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -39,6 +39,7 @@
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option
+from easybuild.tools.filetools import get_cwd
from easybuild.tools.job.backend import JobBackend
from easybuild.tools.utilities import only_if_module_is_available
@@ -320,8 +321,8 @@ def _submit(self):
self.log.debug("Job hold attributes: %s" % hold_attributes[0].value)
# add a bunch of variables (added by qsub)
- # also set PBS_O_WORKDIR to os.getcwd()
- os.environ.setdefault('WORKDIR', os.getcwd())
+ # also set PBS_O_WORKDIR to current working dir
+ os.environ.setdefault('WORKDIR', get_cwd())
defvars = ['MAIL', 'HOME', 'PATH', 'SHELL', 'WORKDIR']
pbsvars = ["PBS_O_%s=%s" % (x, os.environ.get(x, 'NOTFOUND_%s' % x)) for x in defvars]
@@ -484,7 +485,7 @@ def info(self, types=None):
# only expect to have a list with one element
j = jobs[0]
# convert attribs into useable dict
- job_details = dict([(attrib.name, attrib.value) for attrib in j.attribs])
+ job_details = {attrib.name: attrib.value for attrib in j.attribs}
# manually set 'id' attribute
job_details['id'] = j.name
self.log.debug("Found jobinfo %s" % job_details)
diff --git a/easybuild/tools/job/slurm.py b/easybuild/tools/job/slurm.py
index 97a925bd35..517deff56d 100644
--- a/easybuild/tools/job/slurm.py
+++ b/easybuild/tools/job/slurm.py
@@ -1,5 +1,5 @@
##
-# Copyright 2018-2023 Ghent University
+# Copyright 2018-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/loose_version.py b/easybuild/tools/loose_version.py
index 1855fee74a..a454912ed0 100644
--- a/easybuild/tools/loose_version.py
+++ b/easybuild/tools/loose_version.py
@@ -1,21 +1,18 @@
-# This file contains the LooseVersion class based on the class with the same name
-# as present in Python 3.7.4 distutils.
-# The original class is licensed under the Python Software Foundation License Version 2.
-# It was slightly simplified as needed to make it shorter and easier to read.
-# In particular the following changes were made:
-# - Subclass object directly instead of abstract Version class
-# - Fully init the class in the constructor removing the parse method
-# - Always set self.vstring and self.version
-# - Shorten the comparison operators as the NotImplemented case doesn't apply anymore
-# - Changes to documentation and formatting
+"""
+This file contains the LooseVersion class based on the class with the same name
+as present in Python 3.7.4 distutils.
+The original class is licensed under the Python Software Foundation License Version 2.
+It was slightly simplified as needed to make it shorter and easier to read.
+In particular the following changes were made:
+- Subclass object directly instead of abstract Version class
+- Fully init the class in the constructor removing the parse method
+- Always set self.vstring and self.version
+- Shorten the comparison operators as the NotImplemented case doesn't apply anymore
+- Changes to documentation and formatting
+"""
import re
-# Modified: Make this compatible with Python 2
-try:
- from itertools import zip_longest
-except ImportError:
- # Python 2
- from itertools import izip_longest as zip_longest
+from itertools import zip_longest
class LooseVersion(object):
@@ -53,6 +50,22 @@ def version(self):
"""Readonly access to the parsed version (list or None)"""
return self._version
+ def is_prerelease(self, other, markers):
+ """Check if this is a prerelease of other
+
+ Markers is a list of strings that denote a prerelease
+ """
+ if isinstance(other, str):
+ vstring = other
+ else:
+ vstring = other._vstring
+ if self._vstring.startswith(vstring):
+ prerelease = self._vstring[len(vstring):]
+ for marker in markers:
+ if prerelease.startswith(marker):
+ return True
+ return False
+
def __str__(self):
return self._vstring
@@ -64,17 +77,19 @@ def _cmp(self, other):
if isinstance(other, str):
other = LooseVersion(other)
- # Modified: Behave the same in Python 2 & 3 when parts are of different types
- # Taken from https://bugs.python.org/issue14894
- for i, j in zip_longest(self.version, other.version, fillvalue=''):
- if not type(i) is type(j):
+ # Modified: Use string comparison for different types and fill with zeroes/empty strings
+ # Based on https://bugs.python.org/issue14894
+ for i, j in zip_longest(self.version, other.version):
+ if i is None:
+ i = 0 if isinstance(j, int) else ''
+ elif j is None:
+ j = 0 if isinstance(i, int) else ''
+ elif not type(i) is type(j):
i = str(i)
j = str(j)
- if i == j:
- continue
- elif i < j:
+ if i < j:
return -1
- else: # i > j
+ if i > j:
return 1
return 0
diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index 851d4131ff..83a870a77b 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -39,6 +39,7 @@
import os
import re
import tempfile
+from collections import defaultdict
from contextlib import contextmanager
from easybuild.tools import LooseVersion
from textwrap import wrap
@@ -58,7 +59,7 @@ def avail_module_generators():
"""
Return all known module syntaxes.
"""
- return dict([(k.SYNTAX, k) for k in get_subclasses(ModuleGenerator)])
+ return {k.SYNTAX: k for k in get_subclasses(ModuleGenerator)}
def module_generator(app, fake=False):
@@ -153,7 +154,7 @@ def start_module_creation(self):
raise EasyBuildError('Module creation already in process. '
'You cannot create multiple modules at the same time!')
# Mapping of keys/env vars to paths already added
- self.added_paths_per_key = dict()
+ self.added_paths_per_key = defaultdict(set)
txt = self.MODULE_SHEBANG
if txt:
txt += '\n'
@@ -205,14 +206,18 @@ def get_modules_path(self, fake=False, mod_path_suffix=None):
return os.path.join(mod_path, mod_path_suffix)
- def _filter_paths(self, key, paths):
- """Filter out paths already added to key and return the remaining ones"""
+ def _filter_paths(self, key, paths, warn_exists=True):
+ """
+ Filter out paths already added to key and return the remaining ones
+
+ :param warn_exists: Show a warning for paths already added to the key
+ """
if self.added_paths_per_key is None:
# For compatibility this is only a warning for now and we don't filter any paths
print_warning('Module creation has not been started. Call start_module_creation first!')
return paths
- added_paths = self.added_paths_per_key.setdefault(key, set())
+ added_paths = self.added_paths_per_key[key]
# paths can be a string
if isinstance(paths, str):
if paths in added_paths:
@@ -226,15 +231,17 @@ def _filter_paths(self, key, paths):
paths = list(paths)
filtered_paths = [x for x in paths if x not in added_paths and not added_paths.add(x)]
if filtered_paths != paths:
- removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths]
- print_warning("Suppressed adding the following path(s) to $%s of the module as they were already added: %s",
- key, removed_paths,
- log=self.log)
+ if warn_exists:
+ removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths]
+ print_warning("Suppressed adding the following path(s) to $%s of the module "
+ "as they were already added: %s",
+ key, removed_paths,
+ log=self.log)
if not filtered_paths:
filtered_paths = None
return filtered_paths
- def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
+ def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True, delim=':', warn_exists=True):
"""
Generate append-path statements for the given list of paths.
@@ -242,13 +249,16 @@ def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
:param paths: list of paths to append
:param allow_abs: allow providing of absolute paths
:param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+ :param delim: delimiter used between paths
+ :param warn_exists: Show a warning if any path was already added to the variable
"""
- paths = self._filter_paths(key, paths)
+ paths = self._filter_paths(key, paths, warn_exists=warn_exists)
if paths is None:
return ''
- return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
+ return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths,
+ delim=delim)
- def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
+ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True, delim=':', warn_exists=True):
"""
Generate prepend-path statements for the given list of paths.
@@ -256,11 +266,14 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
:param paths: list of paths to append
:param allow_abs: allow providing of absolute paths
:param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+ :param delim: delimiter used between paths
+ :param warn_exists: Show a warning if any path was already added to the variable
"""
- paths = self._filter_paths(key, paths)
+ paths = self._filter_paths(key, paths, warn_exists=warn_exists)
if paths is None:
return ''
- return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
+ return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths,
+ delim=delim)
def _modulerc_check_module_version(self, module_version):
"""
@@ -346,7 +359,7 @@ def modulerc(self, module_version=None, filepath=None, modulerc_txt=None):
module_version_statement = "module-version %(modname)s %(sym_version)s"
- # for Environment Modules we need to guard the module-version statement,
+ # for EnvironmentModulesC we need to guard the module-version statement,
# to avoid "Duplicate version symbol" warning messages where EasyBuild trips over,
# which occur because the .modulerc is parsed twice
# "module-info version " returns its argument if that argument is not a symbolic version (yet),
@@ -483,13 +496,13 @@ def getenv_cmd(self, envvar, default=None):
"""
raise NotImplementedError
- def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None):
+ def load_module(self, mod_name, recursive_unload=False, depends_on=None, unload_modules=None, multi_dep_mods=None):
"""
Generate load statement for specified module.
:param mod_name: name of module to generate load statement for
:param recursive_unload: boolean indicating whether the 'load' statement should be reverted on unload
- :param depends_on: use depends_on statements rather than (guarded) load statements
+ :param depends_on: use depends_on statements rather than (guarded) load statements (DEPRECATED)
:param unload_modules: name(s) of module to unload first
:param multi_dep_mods: list of module names in multi_deps context, to use for guarding load statement
"""
@@ -551,15 +564,16 @@ def unload_module(self, mod_name):
"""
raise NotImplementedError
- def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
+ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True, delim=':'):
"""
Generate prepend-path or append-path statements for the given list of paths.
:param key: environment variable to prepend/append paths to
- :param paths: list of paths to prepend
+ :param paths: list of paths to prepend/append
:param prepend: whether to prepend (True) or append (False) paths
:param allow_abs: allow providing of absolute paths
:param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+ :param delim: delimiter used between paths
"""
raise NotImplementedError
@@ -757,8 +771,18 @@ def check_group(self, group, error_msg=None):
:param group: string with the group name
:param error_msg: error message to print for users outside that group
"""
- self.log.warning("Can't generate robust check in TCL modules for users belonging to group %s.", group)
- return ''
+ if self.modules_tool.supports_tcl_check_group:
+ if error_msg is None:
+ error_msg = "You are not part of '%s' group of users that have access to this software; " % group
+ error_msg += "Please consult with user support how to become a member of this group"
+
+ error_msg = 'error "%s"' % error_msg
+ res = self.conditional_statement('module-info usergroups %s' % group, error_msg, negative=True)
+ else:
+ self.log.warning("Can't generate robust check in Tcl modules for users belonging to group %s.", group)
+ res = ''
+
+ return res
def comment(self, msg):
"""Return string containing given message as a comment."""
@@ -816,19 +840,20 @@ def get_description(self, conflict=True):
"""
Generate a description.
"""
- txt = '\n'.join([
+ lines = [
"proc ModulesHelp { } {",
" puts stderr {%s" % re.sub(r'([{}\[\]])', r'\\\1', self._generate_help_text()),
" }",
'}',
'',
+ ]
+
+ lines.extend([
+ "module-whatis {%s}" % re.sub(r'([{}\[\]])', r'\\\1', line)
+ for line in self._generate_whatis_lines()
])
- lines = [
- '%(whatis_lines)s',
- '',
- "set root %(installdir)s",
- ]
+ lines.extend(['', "set root " + self.app.installdir])
if self.app.cfg['moduleloadnoconflict']:
cond_unload = self.conditional_statement(self.is_loaded('%(name)s'), "module unload %(name)s")
@@ -845,41 +870,36 @@ def get_description(self, conflict=True):
# - 'conflict Compiler/GCC/4.8.2/OpenMPI' for 'Compiler/GCC/4.8.2/OpenMPI/1.6.4'
lines.extend(['', "conflict %s" % os.path.dirname(self.app.short_mod_name)])
- whatis_lines = [
- "module-whatis {%s}" % re.sub(r'([{}\[\]])', r'\\\1', line)
- for line in self._generate_whatis_lines()
- ]
- txt += '\n'.join([''] + lines + ['']) % {
- 'name': self.app.name,
- 'version': self.app.version,
- 'whatis_lines': '\n'.join(whatis_lines),
- 'installdir': self.app.installdir,
- }
-
- return txt
+ return '\n'.join(lines + [''])
def getenv_cmd(self, envvar, default=None):
"""
Return module-syntax specific code to get value of specific environment variable.
"""
if default is None:
- cmd = '$::env(%s)' % envvar
+ if self.modules_tool.supports_tcl_getenv:
+ cmd = '[getenv %s]' % envvar
+ else:
+ cmd = '$::env(%s)' % envvar
else:
- values = {
- 'default': default,
- 'envvar': '::env(%s)' % envvar,
- }
- cmd = '[if { [info exists %(envvar)s] } { concat $%(envvar)s } else { concat "%(default)s" } ]' % values
+ if self.modules_tool.supports_tcl_getenv:
+ cmd = '[getenv %s "%s"]' % (envvar, default)
+ else:
+ values = {
+ 'default': default,
+ 'envvar': '::env(%s)' % envvar,
+ }
+ cmd = '[if { [info exists %(envvar)s] } { concat $%(envvar)s } else { concat "%(default)s" } ]' % values
return cmd
- def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_modules=None, multi_dep_mods=None):
+ def load_module(self, mod_name, recursive_unload=None, depends_on=None, unload_modules=None, multi_dep_mods=None):
"""
Generate load statement for specified module.
:param mod_name: name of module to generate load statement for
:param recursive_unload: boolean indicating whether the 'load' statement should be reverted on unload
(if None: enable if recursive_mod_unload build option or depends_on is True)
- :param depends_on: use depends_on statements rather than (guarded) load statements
+ :param depends_on: use depends_on statements rather than (guarded) load statements (DEPRECATED)
:param unload_modules: name(s) of module to unload first
:param multi_dep_mods: list of module names in multi_deps context, to use for guarding load statement
"""
@@ -888,7 +908,10 @@ def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_
body.extend([self.unload_module(m).strip() for m in unload_modules])
load_template = self.LOAD_TEMPLATE
# Lmod 7.6.1+ supports depends-on which does this most nicely:
- if build_option('mod_depends_on') or depends_on:
+ if (build_option('mod_depends_on') and self.modules_tool.supports_depends_on) or depends_on:
+ if depends_on is not None:
+ depr_msg = "'depends_on' argument of module generator method 'load_module' should not be used anymore"
+ self.log.deprecated(depr_msg, '6.0')
if not self.modules_tool.supports_depends_on:
raise EasyBuildError("depends-on statements in generated module are not supported by modules tool")
load_template = self.LOAD_TEMPLATE_DEPENDS_ON
@@ -899,8 +922,13 @@ def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_
cond_tmpl = None
+ # Environment Modules v4+ safely handles automatic module load by not reloading already
+ # loaded module. No safe guard test is required and it should even be avoided to get the
+ # module dependency correctly tracked.
+ safe_auto_load = self.modules_tool.supports_safe_auto_load
+
if recursive_unload is None:
- recursive_unload = build_option('recursive_mod_unload') or depends_on
+ recursive_unload = build_option('recursive_mod_unload') or depends_on or safe_auto_load
if recursive_unload:
# wrapping the 'module load' statement with an 'is-loaded or mode == unload'
@@ -911,7 +939,7 @@ def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_
# see also http://lmod.readthedocs.io/en/latest/210_load_storms.html
cond_tmpl = "[ module-info mode remove ] || %s"
- if depends_on:
+ if depends_on or safe_auto_load:
if multi_dep_mods and len(multi_dep_mods) > 1:
parent_mod_name = os.path.dirname(mod_name)
guard = self.is_loaded(multi_dep_mods[1:])
@@ -955,15 +983,16 @@ def msg_on_unload(self, msg):
print_cmd = "puts stderr %s" % quote_str(msg, tcl=True)
return '\n'.join(['', self.conditional_statement("module-info mode unload", print_cmd, indent=False)])
- def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
+ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True, delim=':'):
"""
Generate prepend-path or append-path statements for the given list of paths.
:param key: environment variable to prepend/append paths to
- :param paths: list of paths to prepend
+ :param paths: list of paths to prepend/append
:param prepend: whether to prepend (True) or append (False) paths
:param allow_abs: allow providing of absolute paths
:param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+ :param delim: delimiter used between paths
"""
if prepend:
update_type = 'prepend'
@@ -995,7 +1024,8 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
else:
abspaths.append(path)
- statements = ['%s-path\t%s\t\t%s\n' % (update_type, key, p) for p in abspaths]
+ delim_opt = '' if delim == ':' else f' -d "{delim}"'
+ statements = [f'{update_type}-path{delim_opt}\t{key}\t\t{p}\n' for p in abspaths]
return ''.join(statements)
def set_alias(self, key, value):
@@ -1007,7 +1037,7 @@ def set_alias(self, key, value):
def set_as_default(self, module_dir_path, module_version, mod_symlink_paths=None):
"""
- Create a .version file inside the package module folder in order to set the default version for TMod
+ Create a .version file inside the package module folder in order to set the default version
:param module_dir_path: module directory path, e.g. $HOME/easybuild/modules/all/Bison
:param module_version: module version, e.g. 3.0.4
@@ -1146,6 +1176,7 @@ class ModuleGeneratorLua(ModuleGenerator):
PATH_JOIN_TEMPLATE = 'pathJoin(root, "%s")'
UPDATE_PATH_TEMPLATE = '%s_path("%s", %s)'
+ UPDATE_PATH_TEMPLATE_DELIM = '%s_path("%s", %s, "%s")'
START_STR = '[==['
END_STR = ']==]'
@@ -1180,21 +1211,12 @@ def check_group(self, group, error_msg=None):
:param group: string with the group name
:param error_msg: error message to print for users outside that group
"""
- lmod_version = self.modules_tool.version
- min_lmod_version = '6.0.8'
+ if error_msg is None:
+ error_msg = "You are not part of '%s' group of users that have access to this software; " % group
+ error_msg += "Please consult with user support how to become a member of this group"
- if LooseVersion(lmod_version) >= LooseVersion(min_lmod_version):
- if error_msg is None:
- error_msg = "You are not part of '%s' group of users that have access to this software; " % group
- error_msg += "Please consult with user support how to become a member of this group"
-
- error_msg = 'LmodError("' + error_msg + '")'
- res = self.conditional_statement('userInGroup("%s")' % group, error_msg, negative=True)
- else:
- warn_msg = "Can't generate robust check in Lua modules for users belonging to group %s. "
- warn_msg += "Lmod version not recent enough (%s), should be >= %s"
- self.log.warning(warn_msg, group, lmod_version, min_lmod_version)
- res = ''
+ error_msg = 'LmodError("' + error_msg + '")'
+ res = self.conditional_statement('userInGroup("%s")' % group, error_msg, negative=True)
return res
@@ -1261,18 +1283,17 @@ def get_description(self, conflict=True):
"""
Generate a description.
"""
- txt = '\n'.join([
+ lines = [
'help(%s%s' % (self.START_STR, self.check_str(self._generate_help_text())),
'%s)' % self.END_STR,
'',
- ])
-
- lines = [
- "%(whatis_lines)s",
- '',
- 'local root = "%(installdir)s"',
]
+ for line in self._generate_whatis_lines():
+ lines.append("whatis(%s%s%s)" % (self.START_STR, self.check_str(line), self.END_STR))
+
+ lines.extend(['', 'local root = "%s"' % self.app.installdir])
+
if self.app.cfg['moduleloadnoconflict']:
self.log.info("Nothing to do to ensure no conflicts can occur on load when using Lua modules files/Lmod")
@@ -1280,10 +1301,6 @@ def get_description(self, conflict=True):
# conflict on 'name' part of module name (excluding version part at the end)
lines.extend(['', 'conflict("%s")' % os.path.dirname(self.app.short_mod_name)])
- whatis_lines = []
- for line in self._generate_whatis_lines():
- whatis_lines.append("whatis(%s%s%s)" % (self.START_STR, self.check_str(line), self.END_STR))
-
if build_option('module_extensions'):
extensions_list = self._generate_extensions_list()
@@ -1294,15 +1311,7 @@ def get_description(self, conflict=True):
# https://github.com/TACC/Lmod/issues/428
lines.extend(['', self.conditional_statement(self.check_version("8", "2", "8"), extensions_stmt)])
- txt += '\n'.join([''] + lines + ['']) % {
- 'name': self.app.name,
- 'version': self.app.version,
- 'whatis_lines': '\n'.join(whatis_lines),
- 'installdir': self.app.installdir,
- 'homepage': self.app.cfg['homepage'],
- }
-
- return txt
+ return '\n'.join(lines + [''])
def getenv_cmd(self, envvar, default=None):
"""
@@ -1314,14 +1323,14 @@ def getenv_cmd(self, envvar, default=None):
cmd = 'os.getenv("%s") or "%s"' % (envvar, default)
return cmd
- def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_modules=None, multi_dep_mods=None):
+ def load_module(self, mod_name, recursive_unload=None, depends_on=None, unload_modules=None, multi_dep_mods=None):
"""
Generate load statement for specified module.
:param mod_name: name of module to generate load statement for
:param recursive_unload: boolean indicating whether the 'load' statement should be reverted on unload
(if None: enable if recursive_mod_unload build option or depends_on is True)
- :param depends_on: use depends_on statements rather than (guarded) load statements
+ :param depends_on: use depends_on statements rather than (guarded) load statements (DEPRECATED)
:param unload_modules: name(s) of module to unload first
:param multi_dep_mods: list of module names in multi_deps context, to use for guarding load statement
"""
@@ -1331,7 +1340,10 @@ def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_
load_template = self.LOAD_TEMPLATE
# Lmod 7.6+ supports depends_on which does this most nicely:
- if build_option('mod_depends_on') or depends_on:
+ if (build_option('mod_depends_on') and self.modules_tool.supports_depends_on) or depends_on:
+ if depends_on is not None:
+ depr_msg = "'depends_on' argument of module generator method 'load_module' should not be used anymore"
+ self.log.deprecated(depr_msg, '6.0')
if not self.modules_tool.supports_depends_on:
raise EasyBuildError("depends_on statements in generated module are not supported by modules tool")
load_template = self.LOAD_TEMPLATE_DEPENDS_ON
@@ -1426,7 +1438,7 @@ def modulerc(self, module_version=None, filepath=None, modulerc_txt=None):
return super(ModuleGeneratorLua, self).modulerc(module_version=module_version, filepath=filepath,
modulerc_txt=modulerc_txt)
- def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
+ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True, delim=':'):
"""
Generate prepend_path or append_path statements for the given list of paths
@@ -1435,6 +1447,7 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
:param prepend: whether to prepend (True) or append (False) paths
:param allow_abs: allow providing of absolute paths
:param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+ :param delim: delimiter used between paths
"""
if prepend:
update_type = 'prepend'
@@ -1467,7 +1480,10 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
else:
abspaths.append('root')
- statements = [self.UPDATE_PATH_TEMPLATE % (update_type, key, p) for p in abspaths]
+ if delim != ':':
+ statements = [self.UPDATE_PATH_TEMPLATE_DELIM % (update_type, key, p, delim) for p in abspaths]
+ else:
+ statements = [self.UPDATE_PATH_TEMPLATE % (update_type, key, p) for p in abspaths]
statements.append('')
return '\n'.join(statements)
diff --git a/easybuild/tools/module_naming_scheme/__init__.py b/easybuild/tools/module_naming_scheme/__init__.py
index bea5d52dc5..e54f04c31e 100644
--- a/easybuild/tools/module_naming_scheme/__init__.py
+++ b/easybuild/tools/module_naming_scheme/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/module_naming_scheme/categorized_mns.py b/easybuild/tools/module_naming_scheme/categorized_mns.py
index 616baf19c2..cdafc97cbb 100644
--- a/easybuild/tools/module_naming_scheme/categorized_mns.py
+++ b/easybuild/tools/module_naming_scheme/categorized_mns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2016-2023 Ghent University
+# Copyright 2016-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/module_naming_scheme/easybuild_mns.py b/easybuild/tools/module_naming_scheme/easybuild_mns.py
index c12ae9a614..93ff663e77 100644
--- a/easybuild/tools/module_naming_scheme/easybuild_mns.py
+++ b/easybuild/tools/module_naming_scheme/easybuild_mns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py
index 9f2e026eb8..7b368e4cac 100644
--- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py
+++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -120,7 +120,7 @@ def det_toolchain_compilers_name_version(self, tc_comps):
res = None
else:
if len(tc_comps) > 0 and tc_comps[0]:
- comp_versions = dict([(comp['name'], self.det_full_version(comp)) for comp in tc_comps])
+ comp_versions = {comp['name']: self.det_full_version(comp) for comp in tc_comps}
comp_names = comp_versions.keys()
key = ','.join(sorted(comp_names))
if key in COMP_NAME_VERSION_TEMPLATES:
@@ -204,10 +204,10 @@ def det_modpath_extensions(self, ec):
comp_name_ver = None
if ec['name'] in extend_comps:
- for key in COMP_NAME_VERSION_TEMPLATES:
+ for key, comp_tmpl in COMP_NAME_VERSION_TEMPLATES.items():
comp_names = key.split(',')
if ec['name'] in comp_names:
- comp_name, comp_ver_tmpl = COMP_NAME_VERSION_TEMPLATES[key]
+ comp_name, comp_ver_tmpl = comp_tmpl
comp_versions = {ec['name']: self.det_full_version(ec)}
if ec['name'] == 'ifort':
# 'icc' key should be provided since it's the only one used in the template
diff --git a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py
index 8d0c34297f..5603b7db47 100644
--- a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py
+++ b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py
index 0da1c142e3..f282a8c8d8 100644
--- a/easybuild/tools/module_naming_scheme/mns.py
+++ b/easybuild/tools/module_naming_scheme/mns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/module_naming_scheme/toolchain.py b/easybuild/tools/module_naming_scheme/toolchain.py
index 135d996ffb..b85032e6e6 100644
--- a/easybuild/tools/module_naming_scheme/toolchain.py
+++ b/easybuild/tools/module_naming_scheme/toolchain.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py
index e6dac41654..b20c8f394b 100644
--- a/easybuild/tools/module_naming_scheme/utilities.py
+++ b/easybuild/tools/module_naming_scheme/utilities.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -85,7 +85,7 @@ def avail_module_naming_schemes():
import_available_modules('easybuild.tools.module_naming_scheme')
# construct name-to-class dict of available module naming scheme
- avail_mnss = dict([(x.__name__, x) for x in get_subclasses(ModuleNamingScheme)])
+ avail_mnss = {x.__name__: x for x in get_subclasses(ModuleNamingScheme)}
return avail_mnss
diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py
index 75a1022b50..aa2dc7b7e4 100644
--- a/easybuild/tools/modules.py
+++ b/easybuild/tools/modules.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -37,13 +37,14 @@
* Jens Timmerman (Ghent University)
* David Brown (Pacific Northwest National Laboratory)
"""
+import glob
import os
import re
import shlex
from easybuild.base import fancylogger
-from easybuild.tools import StrictVersion
-from easybuild.tools.build_log import EasyBuildError, print_warning
+from easybuild.tools import LooseVersion
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning
from easybuild.tools.config import ERROR, IGNORE, PURGE, UNLOAD, UNSET
from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS
from easybuild.tools.config import build_option, get_modules_tool, install_path
@@ -51,6 +52,7 @@
from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file
from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
from easybuild.tools.run import run_shell_cmd
+from easybuild.tools.systemtools import get_shared_lib_ext
from easybuild.tools.utilities import get_subclasses, nub
# software root/version environment variable name prefixes
@@ -144,11 +146,13 @@ class ModulesTool(object):
COMMAND_SHELL = None
# option to determine the version
VERSION_OPTION = '--version'
- # minimal required version (StrictVersion; suffix rc replaced with b (and treated as beta by StrictVersion))
+ # minimal required version (cannot include -beta or rc)
REQ_VERSION = None
+ # minimal required version to check user's group in modulefile
+ REQ_VERSION_TCL_CHECK_GROUP = None
# deprecated version limit (support for versions below this version is deprecated)
DEPR_VERSION = None
- # maximum version allowed (StrictVersion; suffix rc replaced with b (and treated as beta by StrictVersion))
+ # maximum version allowed (cannot include -beta or rc)
MAX_VERSION = None
# the regexp, should have a "version" group (multiline search)
VERSION_REGEXP = None
@@ -207,6 +211,9 @@ def __init__(self, mod_paths=None, testing=False):
self.check_module_function(allow_mismatch=build_option('allow_modules_tool_mismatch'))
self.set_and_check_version()
self.supports_depends_on = False
+ self.supports_tcl_getenv = False
+ self.supports_tcl_check_group = False
+ self.supports_safe_auto_load = False
def __str__(self):
"""String representation of this ModulesTool instance."""
@@ -239,14 +246,6 @@ def set_and_check_version(self):
if res:
self.version = res.group('version')
self.log.info("Found %s version %s", self.NAME, self.version)
-
- # make sure version is a valid StrictVersion (e.g., 5.7.3.1 is invalid),
- # and replace 'rc' by 'b', to make StrictVersion treat it as a beta-release
- self.version = self.version.replace('rc', 'b').replace('-beta', 'b1')
- if len(self.version.split('.')) > 3:
- self.version = '.'.join(self.version.split('.')[:3])
-
- self.log.info("Converted actual version to '%s'" % self.version)
else:
raise EasyBuildError("Failed to determine %s version from option '%s' output: %s",
self.NAME, self.VERSION_OPTION, txt)
@@ -259,9 +258,10 @@ def set_and_check_version(self):
elif build_option('modules_tool_version_check'):
self.log.debug("Checking whether %s version %s meets requirements", self.NAME, self.version)
+ version = LooseVersion(self.version)
if self.REQ_VERSION is not None:
self.log.debug("Required minimum %s version defined: %s", self.NAME, self.REQ_VERSION)
- if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION):
+ if version < self.REQ_VERSION or version.is_prerelease(self.REQ_VERSION, ['rc', '-beta']):
raise EasyBuildError("EasyBuild requires %s >= v%s, found v%s",
self.NAME, self.REQ_VERSION, self.version)
else:
@@ -269,14 +269,14 @@ def set_and_check_version(self):
if self.DEPR_VERSION is not None:
self.log.debug("Deprecated %s version limit defined: %s", self.NAME, self.DEPR_VERSION)
- if StrictVersion(self.version) < StrictVersion(self.DEPR_VERSION):
+ if version < self.DEPR_VERSION or version.is_prerelease(self.DEPR_VERSION, ['rc', '-beta']):
depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION)
depr_msg += "found version %s" % self.version
self.log.deprecated(depr_msg, '6.0')
if self.MAX_VERSION is not None:
self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION)
- if StrictVersion(self.version) > StrictVersion(self.MAX_VERSION):
+ if self.version > self.MAX_VERSION and not version.is_prerelease(self.MAX_VERSION, ['rc', '-beta']):
raise EasyBuildError("EasyBuild requires %s <= v%s, found v%s",
self.NAME, self.MAX_VERSION, self.version)
else:
@@ -302,10 +302,10 @@ def check_module_function(self, allow_mismatch=False, regex=None):
"""Check whether selected module tool matches 'module' function definition."""
if self.testing:
# grab 'module' function definition from environment if it's there; only during testing
- if 'module' in os.environ:
- output, exit_code = os.environ['module'], 0
- else:
- output, exit_code = None, 1
+ try:
+ output, exit_code = os.environ['module'], EasyBuildExit.SUCCESS
+ except KeyError:
+ output, exit_code = None, EasyBuildExit.FAIL_SYSTEM_CHECK
else:
cmd = "type module"
res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=False, hidden=True, output_file=False)
@@ -316,7 +316,7 @@ def check_module_function(self, allow_mismatch=False, regex=None):
mod_cmd_re = re.compile(regex, re.M)
mod_details = "pattern '%s' (%s)" % (mod_cmd_re.pattern, self.NAME)
- if exit_code == 0:
+ if exit_code == EasyBuildExit.SUCCESS:
if mod_cmd_re.search(output):
self.log.debug("Found pattern '%s' in defined 'module' function." % mod_cmd_re.pattern)
else:
@@ -588,7 +588,8 @@ def mod_exists_via_show(mod_name):
self.log.debug("Skipping warning line '%s'", line)
continue
- # skip lines that start with 'module-' (like 'module-version'),
+ # skip lines that start with 'module-' (like 'module-version')
+ # that may appear with EnvironmentModulesC or EnvironmentModulesTcl,
# see https://github.com/easybuilders/easybuild-framework/issues/3376
if line.startswith('module-'):
self.log.debug("Skipping line '%s' since it starts with 'module-'", line)
@@ -743,7 +744,7 @@ def modulefile_path(self, mod_name, strip_ext=False):
:param mod_name: module name
:param strip_ext: strip (.lua) extension from module fileame (if present)"""
# (possible relative) path is always followed by a ':', and may be prepended by whitespace
- # this works for both environment modules and Lmod
+ # this works for both Environment Modules and Lmod
modpath_re = re.compile(r'^\s*(?P[^/\n]*/[^\s]+):$', re.M)
modpath = self.get_value_from_modulefile(mod_name, modpath_re)
@@ -825,7 +826,7 @@ def run_module(self, *args, **kwargs):
self.log.debug("Output of module command '%s': stdout: %s; stderr: %s", cmd, stdout, stderr)
# also catch and check exit code
- if kwargs.get('check_exit_code', True) and res.exit_code != 0:
+ if kwargs.get('check_exit_code', True) and res.exit_code != EasyBuildExit.SUCCESS:
raise EasyBuildError("Module command '%s' failed with exit code %s; stderr: %s; stdout: %s",
cmd, res.exit_code, stderr, stdout)
@@ -842,6 +843,8 @@ def run_module(self, *args, **kwargs):
# this needs to be taken into account when updating the environment via produced output, see below
# keep track of current values of select env vars, so we can correct the adjusted values below
+ # Identical to `{key: os.environ.get(key, '').split(os.pathsep)[::-1] for key in LD_ENV_VAR_KEYS}`
+ # but Python 2 treats that as a local function and refused the `exec` below
prev_ld_values = dict([(key, os.environ.get(key, '').split(os.pathsep)[::-1]) for key in LD_ENV_VAR_KEYS])
# Change the environment
@@ -945,7 +948,7 @@ def check_loaded_modules(self):
"use the --allow-loaded-modules configuration option.",
"To specify action to take when loaded modules are detected, use %s." % opt,
'',
- "See http://easybuild.readthedocs.io/en/latest/Detecting_loaded_modules.html for more information.",
+ "See https://docs.easybuild.io/detecting-loaded-modules/ for more information.",
])
action = build_option('detect_loaded_modules')
@@ -1117,7 +1120,7 @@ def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps,
if modpath_exts is None:
# only retain dependencies that have a non-empty lists of $MODULEPATH extensions
- modpath_exts = dict([(k, v) for k, v in self.modpath_extensions_for(deps).items() if v])
+ modpath_exts = {k: v for k, v in self.modpath_extensions_for(deps).items() if v}
self.log.debug("Non-empty lists of module path extensions for dependencies: %s" % modpath_exts)
mods_to_top = []
@@ -1148,7 +1151,7 @@ def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps,
path = mods_to_top[:]
if mods_to_top:
# remove retained dependencies from the list, since we're climbing up the module tree
- remaining_modpath_exts = dict([m for m in modpath_exts.items() if not m[0] in mods_to_top])
+ remaining_modpath_exts = {m: v for m, v in modpath_exts.items() if m not in mods_to_top}
self.log.debug("Path to top from %s extended to %s, so recursing to find way to the top",
mod_name, mods_to_top)
@@ -1176,7 +1179,7 @@ def update(self):
class EnvironmentModulesC(ModulesTool):
- """Interface to (C) environment modules (modulecmd)."""
+ """Interface to (C) Environment Modules (modulecmd)."""
NAME = "Environment Modules"
COMMAND = "modulecmd"
REQ_VERSION = '3.2.10'
@@ -1191,7 +1194,7 @@ def run_module(self, *args, **kwargs):
if isinstance(args[0], (list, tuple,)):
args = args[0]
- # some versions of Cray's environment modules tool (3.2.10.x) include a "source */init/bash" command
+ # some versions of Cray's Environment Modules tool (3.2.10.x) include a "source */init/bash" command
# in the output of some "modulecmd python load" calls, which is not a valid Python command,
# which must be stripped out to avoid "invalid syntax" errors when evaluating the output
def tweak_stdout(txt):
@@ -1233,9 +1236,9 @@ def get_setenv_value_from_modulefile(self, mod_name, var_name):
class EnvironmentModulesTcl(EnvironmentModulesC):
- """Interface to (Tcl) environment modules (modulecmd.tcl)."""
+ """Interface to (ancient Tcl-only) Environment Modules (modulecmd.tcl)."""
NAME = "ancient Tcl-only Environment Modules"
- # Tcl environment modules have no --terse (yet),
+ # ancient Tcl-only Environment Modules have no --terse (yet),
# -t must be added after the command ('avail', 'list', etc.)
TERSE_OPTION = (1, '-t')
COMMAND = 'modulecmd.tcl'
@@ -1249,7 +1252,7 @@ class EnvironmentModulesTcl(EnvironmentModulesC):
def set_path_env_var(self, key, paths):
"""Set environment variable with given name to the given list of paths."""
super(EnvironmentModulesTcl, self).set_path_env_var(key, paths)
- # for Tcl environment modules, we need to make sure the _modshare env var is kept in sync
+ # for Tcl Environment Modules, we need to make sure the _modshare env var is kept in sync
setvar('%s_modshare' % key, ':1:'.join(paths), verbose=False)
def run_module(self, *args, **kwargs):
@@ -1312,14 +1315,15 @@ def remove_module_path(self, path, set_mod_paths=True):
self.set_mod_paths()
-class EnvironmentModules(EnvironmentModulesTcl):
- """Interface to environment modules 4.0+"""
+class EnvironmentModules(ModulesTool):
+ """Interface to Environment Modules 4.0+"""
NAME = "Environment Modules"
COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl')
COMMAND_ENVIRONMENT = 'MODULES_CMD'
- REQ_VERSION = '4.0.0'
- DEPR_VERSION = '4.0.0' # needs to be set as EnvironmentModules inherits from EnvironmentModulesTcl
+ REQ_VERSION = '4.3.0'
+ DEPR_VERSION = '4.3.0'
MAX_VERSION = None
+ REQ_VERSION_TCL_CHECK_GROUP = '4.6.0'
VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d[^+\s]*)(\+\S*)?\s'
SHOW_HIDDEN_OPTION = '--all'
@@ -1346,6 +1350,10 @@ def __init__(self, *args, **kwargs):
setvar('MODULES_LIST_TERSE_OUTPUT', '', verbose=False)
super(EnvironmentModules, self).__init__(*args, **kwargs)
+ version = LooseVersion(self.version)
+ self.supports_tcl_getenv = True
+ self.supports_tcl_check_group = version >= LooseVersion(self.REQ_VERSION_TCL_CHECK_GROUP)
+ self.supports_safe_auto_load = True
def check_module_function(self, allow_mismatch=False, regex=None):
"""Check whether selected module tool matches 'module' function definition."""
@@ -1390,7 +1398,7 @@ def available(self, mod_name=None, extra_args=None):
if extra_args is None:
extra_args = []
# make hidden modules visible (requires Environment Modules 4.6.0)
- if StrictVersion(self.version) >= StrictVersion('4.6.0'):
+ if LooseVersion(self.version) >= LooseVersion('4.6.0'):
extra_args.append(self.SHOW_HIDDEN_OPTION)
return super(EnvironmentModules, self).available(mod_name=mod_name, extra_args=extra_args)
@@ -1408,14 +1416,45 @@ def get_setenv_value_from_modulefile(self, mod_name, var_name):
# - line starts with 'setenv'
# - whitespace (spaces & tabs) around variable name
# - curly braces around value if it contain spaces
- value = super(EnvironmentModules, self).get_setenv_value_from_modulefile(mod_name=mod_name,
- var_name=var_name)
+ regex = re.compile(r'^setenv\s+%s\s+(?P.+)' % var_name, re.M)
+ value = self.get_value_from_modulefile(mod_name, regex, strict=False)
if value:
- value = value.strip('{}')
+ value = value.strip(' {}')
return value
+ def remove_module_path(self, path, set_mod_paths=True):
+ """
+ Remove specified module path (using 'module unuse').
+
+ :param path: path to remove from $MODULEPATH via 'unuse'
+ :param set_mod_paths: (re)set self.mod_paths
+ """
+ # remove module path via 'module use' and make sure self.mod_paths is synced
+ # Environment Modules <5.0 keeps track of how often a path was added via 'module use',
+ # so we need to check to make sure it's really removed
+ path = normalize_path(path)
+ while True:
+ try:
+ # Unuse the path that is actually present in the environment
+ module_path = next(p for p in curr_module_paths() if normalize_path(p) == path)
+ except StopIteration:
+ break
+ self.unuse(module_path)
+ if set_mod_paths:
+ self.set_mod_paths()
+
+ def update(self):
+ """Update after new modules were added."""
+
+ version = LooseVersion(self.version)
+ if build_option('update_modules_tool_cache') and version >= LooseVersion('5.3.0'):
+ out = self.run_module('cachebuild', return_stderr=True, check_output=False)
+
+ if self.testing:
+ return out
+
class Lmod(ModulesTool):
"""Interface to Lmod."""
@@ -1438,13 +1477,16 @@ def __init__(self, *args, **kwargs):
setvar('LMOD_REDIRECT', 'no', verbose=False)
# disable extended defaults within Lmod (introduced and set as default in Lmod 8.0.7)
setvar('LMOD_EXTENDED_DEFAULT', 'no', verbose=False)
+ # disabled decorations in "ml --terse avail" output
+ # (introduced in Lmod 8.8, see also https://github.com/TACC/Lmod/issues/690)
+ setvar('LMOD_TERSE_DECORATIONS', 'no', verbose=False)
super(Lmod, self).__init__(*args, **kwargs)
- version = StrictVersion(self.version)
+ version = LooseVersion(self.version)
self.supports_depends_on = True
# See https://lmod.readthedocs.io/en/latest/125_personal_spider_cache.html
- if version >= '8.7.12':
+ if version >= LooseVersion('8.7.12'):
self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.cache', 'lmod')
else:
self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache')
@@ -1645,9 +1687,7 @@ def get_software_root(name, with_env_var=False):
"""
env_var = get_software_root_env_var_name(name)
- root = None
- if env_var in os.environ:
- root = os.getenv(env_var)
+ root = os.getenv(env_var)
if with_env_var:
res = (root, env_var)
@@ -1663,6 +1703,7 @@ def get_software_libdir(name, only_one=True, fs=None):
Returns the library subdirectory, relative to software root.
It fails if multiple library subdirs are found, unless only_one is False which yields a list of all library subdirs.
+ If only_one is True and fs is None, select the one subdirectory with shared or static libraries, if possible.
:param name: name of the software package
:param only_one: indicates whether only one lib path is expected to be found
@@ -1695,6 +1736,16 @@ def get_software_libdir(name, only_one=True, fs=None):
if len(res) == 1:
res = res[0]
else:
+ if fs is None and len(res) == 2:
+ # if both lib and lib64 were found, check if only one (exactly) has libraries;
+ # this is needed for software with library archives in lib64 but other files/directories in lib
+ lib_glob = ['*.%s' % ext for ext in ['a', get_shared_lib_ext()]]
+ has_libs = [any(glob.glob(os.path.join(root, subdir, f)) for f in lib_glob) for subdir in res]
+ if has_libs[0] and not has_libs[1]:
+ return res[0]
+ elif has_libs[1] and not has_libs[0]:
+ return res[1]
+
raise EasyBuildError("Multiple library subdirectories found for %s in %s: %s",
name, root, ', '.join(res))
return res
@@ -1715,9 +1766,7 @@ def get_software_version(name):
"""
env_var = get_software_version_env_var_name(name)
- version = None
- if env_var in os.environ:
- version = os.getenv(env_var)
+ version = os.getenv(env_var)
return version
@@ -1746,7 +1795,7 @@ def avail_modules_tools():
"""
Return all known modules tools.
"""
- class_dict = dict([(x.__name__, x) for x in get_subclasses(ModulesTool)])
+ class_dict = {x.__name__: x for x in get_subclasses(ModulesTool)}
# filter out legacy Modules class
if 'Modules' in class_dict:
del class_dict['Modules']
@@ -1758,7 +1807,7 @@ def avail_modules_tools():
def modules_tool(mod_paths=None, testing=False):
"""
- Return interface to modules tool (environment modules (C, Tcl), or Lmod)
+ Return interface to modules tool (EnvironmentModules, Lmod, ...)
"""
# get_modules_tool might return none (e.g. if config was not initialized yet)
modules_tool = get_modules_tool()
diff --git a/easybuild/tools/multidiff.py b/easybuild/tools/multidiff.py
index 0f8c7f5bac..b8a135bf2f 100644
--- a/easybuild/tools/multidiff.py
+++ b/easybuild/tools/multidiff.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -291,8 +291,8 @@ def multidiff(base, files, colored=True):
offset -= 1
# construct the multi-diff based on the constructed dict
- for line_no in local_diff:
- for (line, filename) in local_diff[line_no]:
+ for line_no, line_infos in local_diff.items():
+ for (line, filename) in line_infos:
mdiff.parse_line(line_no, line.rstrip(), filename, squigly_dict.get(line, '').rstrip())
return str(mdiff)
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 73744c0ddc..77eaaa8fc6 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -59,7 +59,7 @@
from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, get_paths_for
from easybuild.toolchains.compiler.systemcompiler import TC_CONSTANT_SYSTEM
from easybuild.tools import LooseVersion, build_log, run # build_log should always stay there, to ensure EasyBuildLog
-from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError
+from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError, EasyBuildExit
from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror
from easybuild.tools.config import CHECKSUM_PRIORITY_CHOICES, DEFAULT_CHECKSUM_PRIORITY
from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES
@@ -69,27 +69,28 @@
from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL
-from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT
+from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS
from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT
-from easybuild.tools.config import DEFAULT_FILTER_RPATH_SANITY_LIBS
+from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS
from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE
from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS
from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS
from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN
from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path
from easybuild.tools.config import BuildOptions, ConfigurationVariables
+from easybuild.tools.config import PYTHON_SEARCH_PATH_TYPES, PYTHONPATH
from easybuild.tools.configobj import ConfigObj, ConfigObjError
from easybuild.tools.docs import FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT
from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses
from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates
from easybuild.tools.docs import list_easyblocks, list_toolchains
from easybuild.tools.environment import restore_env, unset_env_vars
-from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, install_fake_vsc
-from easybuild.tools.filetools import move_file, which
+from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, get_cwd
+from easybuild.tools.filetools import install_fake_vsc, move_file, which
from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED
from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS
from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS
-from easybuild.tools.github import fetch_easyblocks_from_pr, fetch_github_token
+from easybuild.tools.github import fetch_easyblocks_from_commit, fetch_easyblocks_from_pr, fetch_github_token
from easybuild.tools.hooks import KNOWN_HOOKS
from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains
from easybuild.tools.job.backend import avail_job_backends
@@ -105,6 +106,7 @@
from easybuild.tools.repository.repository import avail_repositories
from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family
from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_os_type, get_system_info
+from easybuild.tools.utilities import flatten
from easybuild.tools.version import this_is_easybuild
@@ -123,8 +125,9 @@ def terminal_supports_colors(stream):
CONFIG_ENV_VAR_PREFIX = 'EASYBUILD'
XDG_CONFIG_HOME = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), ".config"))
-XDG_CONFIG_DIRS = os.environ.get('XDG_CONFIG_DIRS', '/etc').split(os.pathsep)
-DEFAULT_SYS_CFGFILES = [f for d in XDG_CONFIG_DIRS for f in sorted(glob.glob(os.path.join(d, 'easybuild.d', '*.cfg')))]
+XDG_CONFIG_DIRS = os.environ.get('XDG_CONFIG_DIRS', '/etc/xdg').split(os.pathsep)
+DEFAULT_SYS_CFGFILES = [[f for f in sorted(glob.glob(os.path.join(d, 'easybuild.d', '*.cfg')))]
+ for d in XDG_CONFIG_DIRS]
DEFAULT_USER_CFGFILE = os.path.join(XDG_CONFIG_HOME, 'easybuild', 'config.cfg')
DEFAULT_LIST_PR_STATE = GITHUB_PR_STATE_OPEN
@@ -212,7 +215,17 @@ class EasyBuildOptions(GeneralOption):
VERSION = this_is_easybuild()
DEFAULT_LOGLEVEL = 'INFO'
- DEFAULT_CONFIGFILES = DEFAULT_SYS_CFGFILES[:]
+ # https://specifications.freedesktop.org/basedir-spec/latest/
+ # says precedence should be
+ # XDG_CONFIG_HOME > 1st entry of XDG_CONFIG_DIRS > 2nd entry ...
+ # EasyBuild parses this list backwards, gives priority to last entry
+ DEFAULT_CONFIGFILES = flatten(DEFAULT_SYS_CFGFILES[::-1])
+ if 'XDG_CONFIG_DIRS' not in os.environ:
+ old_etc_location = os.path.join('/etc', 'easybuild.d')
+ if os.path.isdir(old_etc_location) and glob.glob(os.path.join(old_etc_location, '*.cfg')):
+ _log.deprecated(f"Using {old_etc_location} is deprecated. Please use "
+ "/etc/xdg/easybuild.d instead or add /etc to XDG_CONFIG_DIRS", '6.0')
+
if os.path.exists(DEFAULT_USER_CFGFILE):
DEFAULT_CONFIGFILES.append(DEFAULT_USER_CFGFILE)
@@ -245,7 +258,7 @@ def __init__(self, *args, **kwargs):
# update or define go_configfiles_initenv in named arguments to pass to parent constructor
go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {})
for section, constants in self.go_cfg_constants.items():
- constants = dict([(name, value) for (name, (value, _)) in constants.items()])
+ constants = {name: value for name, (value, _) in constants.items()}
go_cfg_initenv.setdefault(section, {}).update(constants)
super(EasyBuildOptions, self).__init__(*args, **kwargs)
@@ -403,6 +416,8 @@ def override_options(self):
None, 'store_true', False),
'extra-modules': ("List of extra modules to load after setting up the build environment",
'strlist', 'extend', None),
+ "extra-source-urls": ("Specify URLs to fetch sources from in addition to those in the easyconfig",
+ "urltuple", "add_flex", DEFAULT_EXTRA_SOURCE_URLS, {'metavar': 'URL[|URL]'}),
'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true',
False),
'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, "
@@ -436,11 +451,11 @@ def override_options(self):
"(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None),
'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, "
"(e.g. --hide-toolchains=GCCcore)", 'strlist', 'extend', None),
- 'http-header-fields-urlpat': ("Set extra HTTP header FIELDs when downloading files from URL PATterns. "
- "To not log sensitive values, specify a file containing newline separated "
- "FIELDs. e.g. \"^https://www.example.com::/path/to/headers.txt\" or "
- "\"client[A-z0-9]*.example.com': ['Authorization: Basic token']\".",
- None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FILE|FIELD'}),
+ 'http-header-fields-urlpat': ("Set extra HTTP header FIELD when downloading files from URL PATterns. "
+ "Use FILE (to hide sensitive values) and newline separated FIELDs in the "
+ "same format. e.g. \"^https://www.example.com::path/to/headers.txt\" or "
+ "\"client[A-z0-9]*.example.com:: Authorization: Basic token\".",
+ None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FIELDVALUE|FILE'}),
'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False),
'ignore-test-failure': ("Ignore a failing test step", None, 'store_true', False),
'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False),
@@ -482,6 +497,10 @@ def override_options(self):
None, 'store_true', False),
'pre-create-installdir': ("Create installation directory before submitting build jobs",
None, 'store_true', True),
+ 'prefer-python-search-path': (("Prefer using specified environment variable when possible to specify where"
+ " Python packages were installed; see also "
+ "https://docs.easybuild.io/python-search-path"),
+ 'choice', 'store_or_None', PYTHONPATH, PYTHON_SEARCH_PATH_TYPES),
'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"),
None, 'store_true', False, 'p'),
'read-only-installdir': ("Set read-only permissions on installation directory after installation",
@@ -507,9 +526,18 @@ def override_options(self):
'silence-hook-trigger': ("Suppress printing of debug message every time a hook is triggered",
None, 'store_true', False),
'skip-extensions': ("Skip installation of extensions", None, 'store_true', False),
+ 'skip-sanity-check': ("Skip running the sanity check step "
+ "(e.g. testing for installed files or running basic commands)",
+ None, 'store_true', False),
'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'),
'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False),
+ 'software-commit': (
+ "Git commit to use for the target software build (robot capabilities are automatically disabled)",
+ None, 'store', None),
'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False),
+ 'strict-rpath-sanity-check': ("Perform strict RPATH sanity check, which involces unsetting "
+ "$LD_LIBRARY_PATH before checking whether all required libraries are found",
+ None, 'store_true', False),
'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include",
None, 'store', None),
'trace': ("Provide more information in output to stdout on progress", None, 'store_true', True, 'T'),
@@ -579,9 +607,9 @@ def config_options(self):
'strtuple', 'store', DEFAULT_LOGFILE_FORMAT[:], {'metavar': 'DIR,FORMAT'}),
'module-depends-on': ("Use depends_on (Lmod 7.6.1+) for dependencies in all generated modules "
"(implies recursive unloading of modules).",
- None, 'store_true', False),
+ None, 'store_true', True),
'module-extensions': ("Include 'extensions' statement in generated module file (Lua syntax only)",
- None, 'store_true', False),
+ None, 'store_true', True),
'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS),
'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX,
sorted(avail_module_generators().keys())),
@@ -650,7 +678,8 @@ def informative_options(self):
'check-conflicts': ("Check for version conflicts in dependency graphs", None, 'store_true', False),
'check-eb-deps': ("Check presence and version of (required and optional) EasyBuild dependencies",
None, 'store_true', False),
- 'dep-graph': ("Create dependency graph", None, 'store', None, {'metavar': 'depgraph.'}),
+ 'dep-graph': ("Create dependency graph. Output format depends on , e.g. 'dot', 'png', 'pdf', 'gv'.",
+ None, 'store', None, {'metavar': 'depgraph.'}),
'dump-env-script': ("Dump source script to set up build environment based on toolchain/dependencies",
None, 'store_true', False),
'last-log': ("Print location to EasyBuild log file of last (failed) session", None, 'store_true', False),
@@ -696,10 +725,14 @@ def github_options(self):
'check-style': ("Run a style check on the given easyconfigs", None, 'store_true', False),
'cleanup-easyconfigs': ("Clean up easyconfig files for pull request", None, 'store_true', True),
'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'),
+ 'from-commit': ("Obtain easyconfigs from specified commit", 'str', 'store',
+ None, {'metavar': 'commit_SHA'}),
'from-pr': ("Obtain easyconfigs from specified PR", 'strlist', 'store', [], {'metavar': 'PR#'}),
'git-working-dirs-path': ("Path to Git working directories for EasyBuild repositories", str, 'store', None),
'github-user': ("GitHub username", str, 'store', None),
'github-org': ("GitHub organization", str, 'store', None),
+ 'include-easyblocks-from-commit': ("Include easyblocks from specified commit", 'str', 'store', None,
+ {'metavar': 'commit_SHA'}),
'include-easyblocks-from-pr': ("Include easyblocks from specified PR", 'strlist', 'store', [],
{'metavar': 'PR#'}),
'install-github-token': ("Install GitHub token (requires --github-user)", None, 'store_true', False),
@@ -825,7 +858,7 @@ def job_options(self):
'eb-cmd': ("EasyBuild command to use in jobs", 'str', 'store', DEFAULT_JOB_EB_CMD),
'max-jobs': ("Maximum number of concurrent jobs (queued and running, 0 = unlimited)", 'int', 'store', 0),
'max-walltime': ("Maximum walltime for jobs (in hours)", 'int', 'store', 24),
- 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', os.getcwd()),
+ 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', get_cwd()),
'polling-interval': ("Interval between polls for status of jobs (in seconds)", float, 'store', 30.0),
'target-resource': ("Target resource for jobs", None, 'store', None),
})
@@ -896,7 +929,10 @@ def validate(self):
error_msgs.append(error_msg % (cuda_cc_regex.pattern, ', '.join(faulty_cuda_ccs)))
if error_msgs:
- raise EasyBuildError("Found problems validating the options: %s", '\n'.join(error_msgs))
+ raise EasyBuildError(
+ "Found problems validating the options: %s", '\n'.join(error_msgs),
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
@@ -921,10 +957,6 @@ def postprocess(self):
# set tmpdir
self.tmpdir = set_tmpdir(self.options.tmpdir)
- # early check for opt-in to installing extensions in parallel (experimental feature)
- if self.options.parallel_extensions_install:
- self.log.experimental("installing extensions in parallel")
-
# take --include options into account (unless instructed otherwise)
if self.with_include:
self._postprocess_include()
@@ -984,15 +1016,20 @@ def _postprocess_optarch(self):
n_parts = len(optarch_parts)
map_char_cnts = [p.count(OPTARCH_MAP_CHAR) for p in optarch_parts]
if (n_parts > 1 and any(c != 1 for c in map_char_cnts)) or (n_parts == 1 and map_char_cnts[0] > 1):
- raise EasyBuildError("The optarch option has an incorrect syntax: %s", self.options.optarch)
+ raise EasyBuildError(
+ "The optarch option has an incorrect syntax: %s", self.options.optarch,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
else:
# if there are options for different compilers, we set up a dict
if OPTARCH_MAP_CHAR in optarch_parts[0]:
optarch_dict = {}
for compiler, compiler_opt in [p.split(OPTARCH_MAP_CHAR) for p in optarch_parts]:
if compiler in optarch_dict:
- raise EasyBuildError("The optarch option contains duplicated entries for compiler %s: %s",
- compiler, self.options.optarch)
+ raise EasyBuildError(
+ "The optarch option contains duplicated entries for compiler %s: %s",
+ compiler, self.options.optarch, exit_code=EasyBuildExit.OPTION_ERROR
+ )
else:
optarch_dict[compiler] = compiler_opt
self.options.optarch = optarch_dict
@@ -1004,13 +1041,17 @@ def _postprocess_optarch(self):
def _postprocess_close_pr_reasons(self):
"""Postprocess --close-pr-reasons options"""
if self.options.close_pr_msg:
- raise EasyBuildError("Please either specify predefined reasons with --close-pr-reasons or " +
- "a custom message with--close-pr-msg")
+ raise EasyBuildError(
+ "Please either select a reason with --close-pr-reasons or add a custom message with--close-pr-msg",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
reasons = self.options.close_pr_reasons.split(',')
if any([reason not in VALID_CLOSE_PR_REASONS.keys() for reason in reasons]):
- raise EasyBuildError("Argument to --close-pr_reasons must be a comma separated list of valid reasons " +
- "among %s" % VALID_CLOSE_PR_REASONS.keys())
+ raise EasyBuildError(
+ "Argument to --close-pr_reasons must be a comma separated list of valid reasons among %s",
+ VALID_CLOSE_PR_REASONS.keys(), exit_code=EasyBuildExit.OPTION_ERROR
+ )
self.options.close_pr_msg = ", ".join([VALID_CLOSE_PR_REASONS[reason] for reason in reasons])
def _postprocess_list_prs(self):
@@ -1019,18 +1060,30 @@ def _postprocess_list_prs(self):
nparts = len(list_pr_parts)
if nparts > 3:
- raise EasyBuildError("Argument to --list-prs must be in the format 'state[,order[,direction]]")
+ raise EasyBuildError(
+ "Argument to --list-prs must be in the format 'state[,order[,direction]]",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
list_pr_state = list_pr_parts[0]
list_pr_order = list_pr_parts[1] if nparts > 1 else DEFAULT_LIST_PR_ORDER
list_pr_direc = list_pr_parts[2] if nparts > 2 else DEFAULT_LIST_PR_DIREC
if list_pr_state not in GITHUB_PR_STATES:
- raise EasyBuildError("1st item in --list-prs ('%s') must be one of %s", list_pr_state, GITHUB_PR_STATES)
+ raise EasyBuildError(
+ "1st item in --list-prs ('%s') must be one of %s", list_pr_state, GITHUB_PR_STATES,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
if list_pr_order not in GITHUB_PR_ORDERS:
- raise EasyBuildError("2nd item in --list-prs ('%s') must be one of %s", list_pr_order, GITHUB_PR_ORDERS)
+ raise EasyBuildError(
+ "2nd item in --list-prs ('%s') must be one of %s", list_pr_order, GITHUB_PR_ORDERS,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
if list_pr_direc not in GITHUB_PR_DIRECTIONS:
- raise EasyBuildError("3rd item in --list-prs ('%s') must be one of %s", list_pr_direc, GITHUB_PR_DIRECTIONS)
+ raise EasyBuildError(
+ "3rd item in --list-prs ('%s') must be one of %s", list_pr_direc, GITHUB_PR_DIRECTIONS,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
self.options.list_prs = (list_pr_state, list_pr_order, list_pr_direc)
@@ -1052,41 +1105,68 @@ def _postprocess_checks(self):
# fail early if required dependencies for functionality requiring using GitHub API are not available:
if self.options.from_pr or self.options.include_easyblocks_from_pr or self.options.upload_test_report:
if not HAVE_GITHUB_API:
- raise EasyBuildError("Required support for using GitHub API is not available (see warnings)")
+ raise EasyBuildError(
+ "Required support for using GitHub API is not available (see warnings)",
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# using Lua module syntax only makes sense when modules tool being used is Lmod
if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__:
error_msg = "Generating Lua module files requires Lmod as modules tool; "
mod_syntaxes = ', '.join(sorted(avail_module_generators().keys()))
error_msg += "use --module-syntax to specify a different module syntax to use (%s)" % mod_syntaxes
- raise EasyBuildError(error_msg)
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.OPTION_ERROR)
# check whether specified action --detect-loaded-modules is valid
if self.options.detect_loaded_modules not in LOADED_MODULES_ACTIONS:
error_msg = "Unknown action specified to --detect-loaded-modules: %s (known values: %s)"
- raise EasyBuildError(error_msg % (self.options.detect_loaded_modules, ', '.join(LOADED_MODULES_ACTIONS)))
+ raise EasyBuildError(
+ error_msg, self.options.detect_loaded_modules, ', '.join(LOADED_MODULES_ACTIONS),
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
# make sure a GitHub token is available when it's required
if self.options.upload_test_report:
if not HAVE_KEYRING:
- raise EasyBuildError("Python 'keyring' module required for obtaining GitHub token is not available")
+ raise EasyBuildError(
+ "Python 'keyring' module required for obtaining GitHub token is not available",
+ exit_code=EasyBuildExit.MISSING_EB_DEPENDENCY
+ )
if self.options.github_user is None:
- raise EasyBuildError("No GitHub user name provided, required for fetching GitHub token")
+ raise EasyBuildError(
+ "No GitHub user name provided, required for fetching GitHub token",
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
token = fetch_github_token(self.options.github_user)
if token is None:
- raise EasyBuildError("Failed to obtain required GitHub token for user '%s'" % self.options.github_user)
+ raise EasyBuildError(
+ "Failed to obtain required GitHub token for user '%s'", self.options.github_user,
+ exit_code=EasyBuildExit.FAIL_GITHUB
+ )
# make sure autopep8 is available when it needs to be
if self.options.dump_autopep8:
if not HAVE_AUTOPEP8:
- raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested")
+ raise EasyBuildError(
+ "Python 'autopep8' module required to reformat dumped easyconfigs as requested",
+ exit_code=EasyBuildExit.MISSING_EB_DEPENDENCY
+ )
# if a path is specified to --sysroot, it must exist
if self.options.sysroot:
if os.path.exists(self.options.sysroot):
self.log.info("Specified sysroot '%s' exists: OK", self.options.sysroot)
else:
- raise EasyBuildError("Specified sysroot '%s' does not exist!", self.options.sysroot)
+ raise EasyBuildError(
+ "Specified sysroot '%s' does not exist!", self.options.sysroot,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
+
+ # 'software_commit' is specific to a particular software package and cannot be used with 'robot'
+ if self.options.software_commit:
+ if self.options.robot is not None:
+ print_warning("To allow use of --software-commit robot resolution is being disabled")
+ self.options.robot = None
self.log.info("Checks on configuration options passed")
@@ -1112,7 +1192,10 @@ def _ensure_abs_path(self, opt_name):
setattr(self.options, opt_name, abs_paths)
else:
error_msg = "Don't know how to ensure absolute path(s) for '%s' configuration option (value type: %s)"
- raise EasyBuildError(error_msg, opt_name, type(opt_val))
+ raise EasyBuildError(
+ error_msg, opt_name, type(opt_val),
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
def _postprocess_config(self):
"""Postprocessing of configuration options"""
@@ -1172,7 +1255,10 @@ def _postprocess_config(self):
self.args.append(robot_arg)
self.options.robot = []
else:
- raise EasyBuildError("Argument passed to --robot is not an existing directory: %s", robot_arg)
+ raise EasyBuildError(
+ "Argument passed to --robot is not an existing directory: %s", robot_arg,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
# paths specified to --robot have preference over --robot-paths
# keep both values in sync if robot is enabled, which implies enabling dependency resolver
@@ -1218,8 +1304,9 @@ def _postprocess_list_avail(self):
if self.options.avail_easyconfig_licenses:
msg += avail_easyconfig_licenses(self.options.output_format)
- # dump available easyblocks (unless including easyblocks from pr, in which case it will be done later)
- if self.options.list_easyblocks and not self.options.include_easyblocks_from_pr:
+ # dump available easyblocks (unless including easyblocks from commit or PR, in which case it will be done later)
+ easyblocks_from = self.options.include_easyblocks_from_commit or self.options.include_easyblocks_from_pr
+ if self.options.list_easyblocks and not easyblocks_from:
msg += list_easyblocks(self.options.list_easyblocks, self.options.output_format)
# dump known toolchains
@@ -1263,7 +1350,7 @@ def _postprocess_list_avail(self):
print(msg)
# cleanup tmpdir and exit
- if not self.options.include_easyblocks_from_pr:
+ if not (self.options.include_easyblocks_from_commit or self.options.include_easyblocks_from_pr):
cleanup_and_exit(self.tmpdir)
def avail_repositories(self):
@@ -1308,10 +1395,11 @@ def show_default_configfiles(self):
'',
"* user-level: %s" % os.path.join('${XDG_CONFIG_HOME:-$HOME/.config}', 'easybuild', 'config.cfg'),
" -> %s => %s" % (DEFAULT_USER_CFGFILE, ('not found', 'found')[os.path.exists(DEFAULT_USER_CFGFILE)]),
- "* system-level: %s" % os.path.join('${XDG_CONFIG_DIRS:-/etc}', 'easybuild.d', '*.cfg'),
- " -> %s => %s" % (system_cfg_glob_paths, ', '.join(DEFAULT_SYS_CFGFILES) or "(no matches)"),
+ "* system-level: %s" % os.path.join('${XDG_CONFIG_DIRS:-/etc/xdg}', 'easybuild.d', '*.cfg'),
+ " -> %s => %s" % (system_cfg_glob_paths, ', '.join(flatten(DEFAULT_SYS_CFGFILES)) or "(no matches)"),
'',
- "Default list of existing configuration files (%d): %s" % (found_cfgfile_cnt, found_cfgfile_list),
+ "Default list of existing configuration files (%d, most important last):" % found_cfgfile_cnt,
+ found_cfgfile_list,
]
return '\n'.join(lines)
@@ -1360,9 +1448,9 @@ def show_system_info(self):
'',
"* GPU:",
])
- for vendor in gpu_info:
+ for vendor, vendor_gpu in gpu_info.items():
lines.append(" -> %s" % vendor)
- for gpu, num in gpu_info[vendor].items():
+ for gpu, num in vendor_gpu.items():
lines.append(" -> %sx %s" % (num, gpu))
lines.extend([
@@ -1382,7 +1470,8 @@ def show_config(self):
# options that should never/always be printed
ignore_opts = ['show_config', 'show_full_config']
- include_opts = ['buildpath', 'containerpath', 'installpath', 'repositorypath', 'robot_paths', 'sourcepath']
+ include_opts = ['buildpath', 'containerpath', 'installpath', 'repositorypath', 'robot_paths',
+ 'rpath', 'sourcepath']
cmdline_opts_dict = self.dict_by_prefix()
def reparse_cfg(args=None, withcfg=True):
@@ -1496,11 +1585,31 @@ def parse_options(args=None, with_include=True):
go_args=eb_args, error_env_options=True, error_env_option_method=raise_easybuilderror,
with_include=with_include)
except EasyBuildError as err:
- raise EasyBuildError("Failed to parse configuration options: %s" % err)
+ try:
+ exit_code = err.exit_code
+ except AttributeError:
+ exit_code = EasyBuildExit.OPTION_ERROR
+ raise EasyBuildError("Failed to parse configuration options: %s", err, exit_code=exit_code)
return eb_go
+def check_options(options):
+ """
+ Check configuration options, some combinations are not allowed.
+ """
+ if options.from_commit and options.from_pr:
+ raise EasyBuildError(
+ "--from-commit and --from-pr should not be used together, pick one",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
+
+ if options.include_easyblocks_from_commit and options.include_easyblocks_from_pr:
+ error_msg = "--include-easyblocks-from-commit and --include-easyblocks-from-pr "
+ error_msg += "should not be used together, pick one"
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.OPTION_ERROR)
+
+
def check_root_usage(allow_use_as_root=False):
"""
Check whether we are running as root, and act accordingly
@@ -1518,6 +1627,72 @@ def check_root_usage(allow_use_as_root=False):
"so let's end this here.")
+def handle_include_easyblocks_from(options, log):
+ """
+ Handle --include-easyblocks-from-pr and --include-easyblocks-from-commit
+ """
+ def check_included_multiple(included_easyblocks_from, source):
+ """Check whether easyblock is being included multiple times"""
+ included_multiple = included_easyblocks_from & included_easyblocks
+ if included_multiple:
+ warning_msg = "One or more easyblocks included from multiple locations: %s " \
+ % ', '.join(included_multiple)
+ warning_msg += "(the one(s) from %s will be used)" % source
+ print_warning(warning_msg)
+
+ if options.include_easyblocks_from_pr or options.include_easyblocks_from_commit:
+
+ if options.include_easyblocks:
+ # check if you are including the same easyblock twice
+ included_paths = expand_glob_paths(options.include_easyblocks)
+ included_easyblocks = set([os.path.basename(eb) for eb in included_paths])
+
+ if options.include_easyblocks_from_pr:
+ try:
+ easyblock_prs = [int(x) for x in options.include_easyblocks_from_pr]
+ except ValueError:
+ raise EasyBuildError(
+ "Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
+
+ for easyblock_pr in easyblock_prs:
+ easyblocks_from_pr = fetch_easyblocks_from_pr(easyblock_pr)
+ included_from_pr = set([os.path.basename(eb) for eb in easyblocks_from_pr])
+
+ if options.include_easyblocks:
+ check_included_multiple(included_from_pr, "PR #%s" % easyblock_pr)
+ included_easyblocks |= included_from_pr
+
+ for easyblock in included_from_pr:
+ print_msg("easyblock %s included from PR #%s" % (easyblock, easyblock_pr), log=log)
+
+ include_easyblocks(options.tmpdir, easyblocks_from_pr)
+
+ easyblock_commit = options.include_easyblocks_from_commit
+ if easyblock_commit:
+ easyblocks_from_commit = fetch_easyblocks_from_commit(easyblock_commit)
+ included_from_commit = set([os.path.basename(eb) for eb in easyblocks_from_commit])
+
+ if options.include_easyblocks:
+ check_included_multiple(included_from_commit, "commit %s" % easyblock_commit)
+
+ for easyblock in included_from_commit:
+ print_msg("easyblock %s included from commit %s" % (easyblock, easyblock_commit), log=log)
+
+ include_easyblocks(options.tmpdir, easyblocks_from_commit)
+
+ if options.list_easyblocks:
+ msg = list_easyblocks(options.list_easyblocks, options.output_format)
+ if options.unittest_file:
+ log.info(msg)
+ else:
+ print(msg)
+ # tmpdir is set by option parser via set_tmpdir function
+ tmpdir = tempfile.gettempdir()
+ cleanup_and_exit(tmpdir)
+
+
def set_up_configuration(args=None, logfile=None, testing=False, silent=False, reconfigure=False):
"""
Set up EasyBuild configuration, by parsing configuration settings & initialising build options.
@@ -1538,6 +1713,8 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r
eb_go = parse_options(args=args)
options = eb_go.options
+ check_options(options)
+
# tmpdir is set by option parser via set_tmpdir function
tmpdir = tempfile.gettempdir()
@@ -1572,20 +1749,27 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r
try:
from_prs = [int(x) for x in eb_go.options.from_pr]
except ValueError:
- raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.")
+ raise EasyBuildError(
+ "Argument to --from-pr must be a comma separated list of PR #s.",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
try:
review_pr = (lambda x: int(x) if x else None)(eb_go.options.review_pr)
except ValueError:
- raise EasyBuildError("Argument to --review-pr must be an integer PR #.")
+ raise EasyBuildError(
+ "Argument to --review-pr must be an integer PR #.",
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
tweaked_ecs = try_to_generate and build_specs
- tweaked_ecs_paths, pr_paths = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_prs=from_prs,
- review_pr=review_pr)
+ tweaked_ecs_paths, extra_ec_paths = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_prs=from_prs,
+ from_commit=eb_go.options.from_commit,
+ review_pr=review_pr)
auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query
- robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_paths, auto_robot=auto_robot)
+ robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, extra_ec_paths, auto_robot=auto_robot)
log.debug("Full robot path: %s", robot_path)
if not robot_path:
@@ -1599,7 +1783,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r
'build_specs': build_specs,
'command_line': eb_cmd_line,
'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata),
- 'pr_paths': pr_paths,
+ 'extra_ec_paths': extra_ec_paths,
'robot_path': robot_path,
'silent': testing or new_update_opt,
'try_to_generate': try_to_generate,
@@ -1625,41 +1809,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r
init_build_options(build_options=build_options, cmdline_options=options)
# done here instead of in _postprocess_include because github integration requires build_options to be initialized
- if eb_go.options.include_easyblocks_from_pr:
- try:
- easyblock_prs = [int(x) for x in eb_go.options.include_easyblocks_from_pr]
- except ValueError:
- raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.")
-
- if eb_go.options.include_easyblocks:
- # check if you are including the same easyblock twice
- included_paths = expand_glob_paths(eb_go.options.include_easyblocks)
- included_from_file = set([os.path.basename(eb) for eb in included_paths])
-
- for easyblock_pr in easyblock_prs:
- easyblocks_from_pr = fetch_easyblocks_from_pr(easyblock_pr)
- included_from_pr = set([os.path.basename(eb) for eb in easyblocks_from_pr])
-
- if eb_go.options.include_easyblocks:
- included_twice = included_from_pr & included_from_file
- if included_twice:
- warning_msg = "One or more easyblocks included from multiple locations: %s " \
- % ', '.join(included_twice)
- warning_msg += "(the one(s) from PR #%s will be used)" % easyblock_pr
- print_warning(warning_msg)
-
- for easyblock in included_from_pr:
- print_msg("easyblock %s included from PR #%s" % (easyblock, easyblock_pr), log=log)
-
- include_easyblocks(eb_go.options.tmpdir, easyblocks_from_pr)
-
- if eb_go.options.list_easyblocks:
- msg = list_easyblocks(eb_go.options.list_easyblocks, eb_go.options.output_format)
- if eb_go.options.unittest_file:
- log.info(msg)
- else:
- print(msg)
- cleanup_and_exit(tmpdir)
+ handle_include_easyblocks_from(eb_go.options, log)
check_python_version()
@@ -1785,7 +1935,10 @@ def parse_external_modules_metadata(cfgs):
paths.extend(res)
else:
# if there are no matches, we report an error to avoid silently ignores faulty paths
- raise EasyBuildError("Specified path for file with external modules metadata does not exist: %s", cfg)
+ raise EasyBuildError(
+ "Specified path for file with external modules metadata does not exist: %s", cfg,
+ exit_code=EasyBuildExit.OPTION_ERROR
+ )
cfgs = paths
# use external modules metadata configuration files that are available by default, unless others are specified
@@ -1817,7 +1970,10 @@ def parse_external_modules_metadata(cfgs):
try:
parsed_metadata.merge(ConfigObj(cfg))
except ConfigObjError as err:
- raise EasyBuildError("Failed to parse %s with external modules metadata: %s", cfg, err)
+ raise EasyBuildError(
+ "Failed to parse %s with external modules metadata: %s", cfg, err,
+ exit_code=EasyBuildExit.MODULE_ERROR
+ )
known_metadata_keys = ['name', 'prefix', 'version']
unknown_keys = {}
@@ -1838,14 +1994,17 @@ def parse_external_modules_metadata(cfgs):
# if both names and versions are available, lists must be of same length
names, versions = entry.get('name'), entry.get('version')
if names is not None and versions is not None and len(names) != len(versions):
- raise EasyBuildError("Different length for lists of names/versions in metadata for external module %s: "
- "names: %s; versions: %s", mod, names, versions)
+ raise EasyBuildError(
+ "Different length for lists of names/versions in metadata for external module %s: ; "
+ "names: %s; versions: %s", mod, names, versions,
+ exit_code=EasyBuildExit.MODULE_ERROR
+ )
if unknown_keys:
error_msg = "Found metadata entries with unknown keys:"
for mod in sorted(unknown_keys.keys()):
error_msg += "\n* %s: %s" % (mod, ', '.join(sorted(unknown_keys[mod])))
- raise EasyBuildError(error_msg)
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.MODULE_ERROR)
_log.debug("External modules metadata: %s", parsed_metadata)
return parsed_metadata
@@ -1898,7 +2057,7 @@ def set_tmpdir(tmpdir=None, raise_error=False):
os.chmod(tmptest_file, 0o700)
res = run_shell_cmd(tmptest_file, fail_on_error=False, in_dry_run=True, hidden=True, stream_output=False,
with_hooks=False)
- if res.exit_code:
+ if res.exit_code != EasyBuildExit.SUCCESS:
msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir()
msg += "This can cause problems in the build process, consider using --tmpdir."
if raise_error:
@@ -1931,6 +2090,7 @@ def opts_dict_to_eb_opts(args_dict):
:return: a list of strings representing command-line options for the 'eb' command
"""
+ allow_multiple_calls = ['amend', 'try-amend']
_log.debug("Converting dictionary %s to argument list" % args_dict)
args = []
for arg in sorted(args_dict):
@@ -1940,14 +2100,18 @@ def opts_dict_to_eb_opts(args_dict):
prefix = '--'
option = prefix + str(arg)
value = args_dict[arg]
- if isinstance(value, (list, tuple)):
- value = ','.join(str(x) for x in value)
- if value in [True, None]:
+ if str(arg) in allow_multiple_calls:
+ if not isinstance(value, (list, tuple)):
+ value = [value]
+ args.extend(option + '=' + str(x) for x in value)
+ elif value in [True, None]:
args.append(option)
elif value is False:
args.append('--disable-' + option[2:])
elif value is not None:
+ if isinstance(value, (list, tuple)):
+ value = ','.join(str(x) for x in value)
args.append(option + '=' + str(value))
_log.debug("Converted dictionary %s to argument list %s" % (args_dict, args))
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index c10b169e57..703fd4576a 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# #
-# Copyright 2021-2023 Ghent University
+# Copyright 2021-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -33,6 +33,7 @@
"""
import functools
from collections import OrderedDict
+import sys
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style
@@ -328,9 +329,7 @@ def print_checks(checks_data):
if use_rich():
console = Console()
- # don't use console.print, which causes SyntaxError in Python 2
- console_print = getattr(console, 'print') # noqa: B009
- console_print('')
+ console.print('')
for section in checks_data:
section_checks = checks_data[section]
@@ -382,11 +381,25 @@ def print_checks(checks_data):
lines.append('')
if use_rich():
- console_print(table)
+ console.print(table)
else:
print('\n'.join(lines))
+def print_error(error_msg, rich_highlight=True):
+ """
+ Print error message, using a Rich Console instance if possible.
+ Newlines before/after message are automatically added.
+
+ :param rich_highlight: boolean indicating whether automatic highlighting by Rich should be enabled
+ """
+ if use_rich():
+ console = Console(stderr=True)
+ console.print('\n\n' + error_msg + '\n', highlight=rich_highlight)
+ else:
+ sys.stderr.write('\n' + error_msg + '\n\n')
+
+
# this constant must be defined at the end, since functions used as values need to be defined
PROGRESS_BAR_TYPES = {
PROGRESS_BAR_DOWNLOAD_ALL: download_all_progress_bar,
diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py
index 9ae9b37419..033687fac1 100644
--- a/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py
+++ b/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- vim: set fileencoding=utf-8
##
-# Copyright 2015-2022 Ghent University
+# Copyright 2015-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py
index a6543a7d53..7a53a86101 100644
--- a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py
+++ b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/package/package_naming_scheme/pns.py b/easybuild/tools/package/package_naming_scheme/pns.py
index 081a94b073..8683ae8d37 100644
--- a/easybuild/tools/package/package_naming_scheme/pns.py
+++ b/easybuild/tools/package/package_naming_scheme/pns.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py
index adaa0814fc..ea18c2b206 100644
--- a/easybuild/tools/package/utilities.py
+++ b/easybuild/tools/package/utilities.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -58,7 +58,7 @@ def avail_package_naming_schemes():
They are loaded from the easybuild.package.package_naming_scheme namespace
"""
import_available_modules('easybuild.tools.package.package_naming_scheme')
- class_dict = dict([(x.__name__, x) for x in get_subclasses(PackageNamingScheme)])
+ class_dict = {x.__name__: x for x in get_subclasses(PackageNamingScheme)}
return class_dict
diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py
index 6c94517c69..b25d8b9bdc 100644
--- a/easybuild/tools/parallelbuild.py
+++ b/easybuild/tools/parallelbuild.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -43,6 +43,7 @@
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option, get_repository, get_repositorypath
+from easybuild.tools.filetools import get_cwd
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
from easybuild.tools.job.backend import job_backend
from easybuild.tools.repository.repository import init_repository
@@ -126,7 +127,7 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True):
:param testing: If `True`, skip actual job submission
:param prepare_first: prepare by runnning fetch step first for each easyconfig
"""
- curdir = os.getcwd()
+ curdir = get_cwd()
# regex pattern for options to ignore (help options can't reach here)
ignore_opts = re.compile('^--robot$|^--job|^--try-.*$|^--easystack$')
diff --git a/easybuild/tools/py2vs3/__init__.py b/easybuild/tools/py2vs3/__init__.py
index 4bd0e47191..d0518f2271 100644
--- a/easybuild/tools/py2vs3/__init__.py
+++ b/easybuild/tools/py2vs3/__init__.py
@@ -1,5 +1,5 @@
#
-# Copyright 2019-2023 Ghent University
+# Copyright 2019-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py
index 21dfc49e10..2f29f0fd54 100644
--- a/easybuild/tools/py2vs3/py3.py
+++ b/easybuild/tools/py2vs3/py3.py
@@ -1,5 +1,5 @@
#
-# Copyright 2019-2023 Ghent University
+# Copyright 2019-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -36,7 +36,6 @@
import json
import sys
import urllib.request as std_urllib # noqa
-from collections import OrderedDict # noqa
from collections.abc import Mapping # noqa
from functools import cmp_to_key
from importlib.util import spec_from_file_location, module_from_spec
diff --git a/easybuild/tools/repository/filerepo.py b/easybuild/tools/repository/filerepo.py
index a35c7054f0..6161b7f37b 100644
--- a/easybuild/tools/repository/filerepo.py
+++ b/easybuild/tools/repository/filerepo.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/repository/gitrepo.py b/easybuild/tools/repository/gitrepo.py
index 97fc6c33da..feaf8cb2ad 100644
--- a/easybuild/tools/repository/gitrepo.py
+++ b/easybuild/tools/repository/gitrepo.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/repository/hgrepo.py b/easybuild/tools/repository/hgrepo.py
index 8cdf145b52..83a7890cc3 100644
--- a/easybuild/tools/repository/hgrepo.py
+++ b/easybuild/tools/repository/hgrepo.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py
index 743a82bc36..1bbde0178a 100644
--- a/easybuild/tools/repository/repository.py
+++ b/easybuild/tools/repository/repository.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -145,7 +145,7 @@ def avail_repositories(check_useable=True):
"""
import_available_modules('easybuild.tools.repository')
- class_dict = dict([(x.__name__, x) for x in get_subclasses(Repository) if x.USABLE or not check_useable])
+ class_dict = {x.__name__: x for x in get_subclasses(Repository) if x.USABLE or not check_useable}
if 'FileRepository' not in class_dict:
raise EasyBuildError("avail_repositories: FileRepository missing from list of repositories")
diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py
index 6c955c7fd0..1e5d980f3b 100644
--- a/easybuild/tools/repository/svnrepo.py
+++ b/easybuild/tools/repository/svnrepo.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -46,7 +46,7 @@
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import remove_dir
+from easybuild.tools.filetools import get_cwd, remove_dir
from easybuild.tools.repository.filerepo import FileRepository
from easybuild.tools.utilities import only_if_module_is_available
@@ -145,7 +145,7 @@ def stage_file(self, path):
"""
if self.client and not self.client.status(path)[0].is_versioned:
# add it to version control
- self.log.debug("Going to add %s (working copy: %s, cwd %s)" % (path, self.wc, os.getcwd()))
+ self.log.debug("Going to add %s (working copy: %s, cwd %s)" % (path, self.wc, get_cwd()))
self.client.add(path)
def add_easyconfig(self, cfg, name, version, stats, previous_stats):
diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py
index 68a4c9028b..c2c9729209 100644
--- a/easybuild/tools/robot.py
+++ b/easybuild/tools/robot.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -43,9 +43,9 @@
from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, process_easyconfig
from easybuild.framework.easyconfig.easyconfig import robot_find_easyconfig, verify_easyconfig_filename
from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit
from easybuild.tools.config import build_option
-from easybuild.tools.filetools import det_common_path_prefix, search_file
+from easybuild.tools.filetools import det_common_path_prefix, get_cwd, search_file
from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
from easybuild.tools.utilities import flatten, nub
@@ -54,7 +54,7 @@
_log = fancylogger.getLogger('tools.robot', fname=False)
-def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_paths, auto_robot=False):
+def det_robot_path(robot_paths_option, tweaked_ecs_paths, extra_ec_paths, auto_robot=False):
"""Determine robot path."""
robot_path = robot_paths_option[:]
_log.info("Using robot path(s): %s", robot_path)
@@ -70,9 +70,9 @@ def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_paths, auto_robot=F
_log.info("Prepended list of robot search paths with %s and appended with %s: %s", tweaked_ecs_path,
tweaked_ecs_deps_path, robot_path)
- if pr_paths is not None:
- robot_path.extend(pr_paths)
- _log.info("Extended list of robot search paths with %s: %s", pr_paths, robot_path)
+ if extra_ec_paths is not None:
+ robot_path.extend(extra_ec_paths)
+ _log.info("Extended list of robot search paths with %s: %s", extra_ec_paths, robot_path)
return robot_path
@@ -242,7 +242,7 @@ def dry_run(easyconfigs, modtool, short=False):
:param short: use short format for overview: use a variable for common prefixes
"""
lines = []
- if build_option('robot_path') is None:
+ if build_option('robot') is None:
lines.append("Dry run: printing build status of easyconfigs")
all_specs = easyconfigs
else:
@@ -288,16 +288,18 @@ def dry_run(easyconfigs, modtool, short=False):
return '\n'.join(lines)
-def missing_deps(easyconfigs, modtool):
+def missing_deps(easyconfigs, modtool, terse=False):
"""
Determine subset of easyconfigs for which no module is installed yet.
"""
ordered_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False)
missing = skip_available(ordered_ecs, modtool)
- if missing:
+ if terse:
+ lines = [os.path.basename(x['ec'].path) for x in missing]
+ elif missing:
lines = ['', "%d out of %d required modules missing:" % (len(missing), len(ordered_ecs)), '']
- for ec in [x['ec'] for x in missing]:
+ for ec in (x['ec'] for x in missing):
if ec.short_mod_name != ec.full_mod_name:
modname = '%s | %s' % (ec.mod_subdir, ec.short_mod_name)
else:
@@ -323,7 +325,7 @@ def raise_error_missing_deps(missing_deps, extra_msg=None):
error_msg = "Missing dependencies: %s" % mod_names
if extra_msg:
error_msg += ' (%s)' % extra_msg
- raise EasyBuildError(error_msg)
+ raise EasyBuildError(error_msg, exit_code=EasyBuildExit.MISSING_DEPENDENCY)
def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_error_missing_ecs=True):
@@ -489,7 +491,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, con
"""
search_path = build_option('robot_path')
if not search_path:
- search_path = [os.getcwd()]
+ search_path = [get_cwd()]
extra_search_paths = build_option('search_paths')
# If we're returning a list of possible resolutions by the robot, don't include the extra_search_paths
if extra_search_paths and consider_extra_paths:
diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py
index 2592e198f1..4911ae4e8e 100644
--- a/easybuild/tools/run.py
+++ b/easybuild/tools/run.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -35,21 +35,26 @@
* Toon Willems (Ghent University)
* Ward Poelmans (Ghent University)
"""
-import contextlib
+import fcntl
import functools
import inspect
+import locale
import os
import re
-import signal
+import shlex
import shutil
import string
import subprocess
-import sys
import tempfile
import time
from collections import namedtuple
from datetime import datetime
+# import deprecated functions so they can still be imported from easybuild.tools.run, for now
+from easybuild._deprecated import check_async_cmd, check_log_for_errors, complete_cmd, extract_errors_from_log # noqa
+from easybuild._deprecated import get_output_from_process, parse_cmd_output, parse_log_for_error # noqa
+from easybuild._deprecated import run_cmd, run_cmd_qa # noqa
+
try:
# get_native_id is only available in Python >= 3.8
from threading import get_native_id as get_thread_id
@@ -57,24 +62,19 @@
# get_ident is available in Python >= 3.3
from threading import get_ident as get_thread_id
-import easybuild.tools.asyncprocess as asyncprocess
from easybuild.base import fancylogger
-from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since
-from easybuild.tools.config import ERROR, IGNORE, WARN, build_option
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, CWD_NOTFOUND_ERROR
+from easybuild.tools.build_log import dry_run_msg, print_msg, time_str_since
+from easybuild.tools.config import build_option
from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook
-from easybuild.tools.utilities import nub, trace_msg
+from easybuild.tools.output import COLOR_RED, COLOR_YELLOW, colorize, print_error
+from easybuild.tools.utilities import trace_msg
_log = fancylogger.getLogger('run', fname=False)
-errors_found_in_log = 0
-
-# default strictness level
-strictness = WARN
-
-
-CACHED_COMMANDS = [
+CACHED_COMMANDS = (
"sysctl -n hw.cpufrequency_max", # used in get_cpu_speed (OS X)
"sysctl -n hw.memsize", # used in get_total_memory (OS X)
"sysctl -n hw.ncpu", # used in get_avail_core_count (OS X)
@@ -83,11 +83,23 @@
"type module", # used in ModulesTool.check_module_function
"type _module_raw", # used in EnvironmentModules.check_module_function
"ulimit -u", # used in det_parallelism
-]
-
+)
RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir',
- 'out_file', 'err_file', 'thread_id', 'task_id'))
+ 'out_file', 'err_file', 'cmd_sh', 'thread_id', 'task_id'))
+RunShellCmdResult.__doc__ = """A namedtuple that represents the result of a call to run_shell_cmd,
+with the following fields:
+- cmd: the command that was executed;
+- exit_code: the exit code of the command (zero if it was successful, non-zero if not);
+- output: output of the command (stdout+stderr combined, only stdout if stderr was caught separately);
+- stderr: stderr output produced by the command, if caught separately (None otherwise);
+- work_dir: the working directory of the command;
+- out_file: path to file with output of command (stdout+stderr combined, only stdout if stderr was caught separately);
+- err_file: path to file with stderr output of command, if caught separately (None otherwise);
+- cmd_sh: path to script to set up interactive shell with environment in which command was executed;
+- thread_id: thread ID of command that was executed (None unless asynchronous mode was enabled for running command);
+- task_id: task ID of command, if it was specified (None otherwise);
+"""
class RunShellCmdError(BaseException):
@@ -102,6 +114,7 @@ def __init__(self, cmd_result, caller_info, *args, **kwargs):
self.out_file = cmd_result.out_file
self.stderr = cmd_result.stderr
self.err_file = cmd_result.err_file
+ self.cmd_sh = cmd_result.cmd_sh
self.caller_info = caller_info
@@ -113,33 +126,36 @@ def print(self):
Report failed shell command for this RunShellCmdError instance
"""
- def pad_4_spaces(msg):
- return ' ' * 4 + msg
+ def pad_4_spaces(msg, color=None):
+ padded_msg = ' ' * 4 + msg
+ if color:
+ return colorize(padded_msg, color)
+ else:
+ return padded_msg
+
+ caller_file_name, caller_line_nr, caller_function_name = self.caller_info
+ called_from_info = f"'{caller_function_name}' function in {caller_file_name} (line {caller_line_nr})"
error_info = [
- '',
- "ERROR: Shell command failed!",
+ colorize("ERROR: Shell command failed!", COLOR_RED),
pad_4_spaces(f"full command -> {self.cmd}"),
pad_4_spaces(f"exit code -> {self.exit_code}"),
+ pad_4_spaces(f"called from -> {called_from_info}"),
pad_4_spaces(f"working directory -> {self.work_dir}"),
]
if self.out_file is not None:
# if there's no separate file for error/warnings, then out_file includes both stdout + stderr
out_info_msg = "output (stdout + stderr)" if self.err_file is None else "output (stdout) "
- error_info.append(pad_4_spaces(f"{out_info_msg} -> {self.out_file}"))
+ error_info.append(pad_4_spaces(f"{out_info_msg} -> {self.out_file}", color=COLOR_YELLOW))
if self.err_file is not None:
- error_info.append(pad_4_spaces(f"error/warnings (stderr) -> {self.err_file}"))
+ error_info.append(pad_4_spaces(f"error/warnings (stderr) -> {self.err_file}", color=COLOR_YELLOW))
- caller_file_name, caller_line_nr, caller_function_name = self.caller_info
- called_from_info = f"'{caller_function_name}' function in {caller_file_name} (line {caller_line_nr})"
- error_info.extend([
- pad_4_spaces(f"called from -> {called_from_info}"),
- '',
- ])
+ if self.cmd_sh is not None:
+ error_info.append(pad_4_spaces(f"interactive shell script -> {self.cmd_sh}", color=COLOR_YELLOW))
- sys.stderr.write('\n'.join(error_info) + '\n')
+ print_error('\n'.join(error_info), rich_highlight=False)
def raise_run_shell_cmd_error(cmd_res):
@@ -151,7 +167,7 @@ def raise_run_shell_cmd_error(cmd_res):
# need to go 3 levels down:
# 1) this function
# 2) run_shell_cmd function
- # 3) run_cmd_cache decorator
+ # 3) run_shell_cmd_cache decorator
# 4) actual caller site
frameinfo = inspect.getouterframes(inspect.currentframe())[3]
caller_info = (frameinfo.filename, frameinfo.lineno, frameinfo.function)
@@ -159,15 +175,15 @@ def raise_run_shell_cmd_error(cmd_res):
raise RunShellCmdError(cmd_res, caller_info)
-def run_cmd_cache(func):
+def run_shell_cmd_cache(func):
"""Function decorator to cache (and retrieve cached) results of running commands."""
cache = {}
@functools.wraps(func)
def cache_aware_func(cmd, *args, **kwargs):
"""Retrieve cached result of selected commands, or run specified and collect & cache result."""
- # cache key is combination of command and input provided via stdin ('stdin' for run, 'inp' for run_cmd)
- key = (cmd, kwargs.get('stdin', None) or kwargs.get('inp', None))
+ # cache key is combination of command and input provided via stdin
+ key = (cmd, kwargs.get('stdin', None))
# fetch from cache if available, cache it if it's not, but only on cmd strings
if isinstance(cmd, str) and key in cache:
_log.debug("Using cached value for command '%s': %s", cmd, cache[key])
@@ -185,9 +201,6 @@ def cache_aware_func(cmd, *args, **kwargs):
return cache_aware_func
-run_shell_cmd_cache = run_cmd_cache
-
-
def fileprefix_from_cmd(cmd, allowed_chars=False):
"""
Simplify the cmd to only the allowed_chars we want in a filename
@@ -201,11 +214,154 @@ def fileprefix_from_cmd(cmd, allowed_chars=False):
return ''.join([c for c in cmd if c in allowed_chars])
+def create_cmd_scripts(cmd_str, work_dir, env, tmpdir, out_file, err_file):
+ """
+ Create helper scripts for specified command in specified directory:
+ - env.sh which can be sourced to define environment in which command was run;
+ - cmd.sh to create interactive (bash) shell session with working directory and environment,
+ and with the command in shell history;
+ """
+ # Save environment variables in env.sh which can be sourced to restore environment
+ if env is None:
+ env = os.environ.copy()
+
+ # Decode any declared bash functions
+ proc = subprocess.Popen('declare -f', stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+ env=env, shell=True, executable='bash')
+ (bash_functions, _) = proc.communicate()
+
+ env_fp = os.path.join(tmpdir, 'env.sh')
+ with open(env_fp, 'w') as fid:
+ # unset all environment variables in current environment first to start from a clean slate;
+ # we need to be careful to filter out functions definitions, so first undefine those
+ fid.write('\n'.join([
+ 'for var in $(compgen -e); do',
+ ' unset "$var"',
+ 'done',
+ ]) + '\n')
+ # also unset any bash functions
+ fid.write('\n'.join([
+ 'for func in $(compgen -A function); do',
+ ' if [[ $func != _* ]]; then',
+ ' unset -f "$func"',
+ ' fi',
+ 'done',
+ ]) + '\n')
+
+ # excludes bash functions (environment variables ending with %)
+ fid.write('\n'.join(f'export {key}={shlex.quote(value)}' for key, value in sorted(env.items())
+ if not key.endswith('%')) + '\n')
+
+ fid.write(bash_functions.decode(errors='ignore') + '\n')
+
+ fid.write('\n\nPS1="eb-shell> "')
+
+ # define $EB_CMD_OUT_FILE to contain path to file with command output
+ fid.write(f'\nEB_CMD_OUT_FILE="{out_file}"')
+ # define $EB_CMD_ERR_FILE to contain path to file with command stderr output (if available)
+ if err_file:
+ fid.write(f'\nEB_CMD_ERR_FILE="{err_file}"')
+
+ # also change to working directory (to ensure that working directory is correct for interactive bash shell)
+ fid.write(f'\ncd "{work_dir}"')
+
+ # reset shell history to only include executed command
+ fid.write(f'\nhistory -s {shlex.quote(cmd_str)}')
+
+ # Make script that sets up bash shell with specified environment and working directory
+ cmd_fp = os.path.join(tmpdir, 'cmd.sh')
+ with open(cmd_fp, 'w') as fid:
+ fid.write('#!/usr/bin/env bash\n')
+ fid.write('# Run this script to set up a shell environment that EasyBuild used to run the shell command\n')
+ fid.write('\n'.join([
+ 'EB_SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )',
+ f'echo "# Shell for the command: {shlex.quote(cmd_str)}"',
+ 'echo "# Use command history, exit to stop"',
+ # using -i to force interactive shell, so env.sh is also sourced when -c is used to run commands
+ 'bash --rcfile $EB_SCRIPT_DIR/env.sh -i "$@"',
+ ]))
+ os.chmod(cmd_fp, 0o775)
+
+ return cmd_fp
+
+
+def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns):
+ """
+ Private helper function to try and answer questions raised in interactive shell commands.
+ """
+ match_found = False
+
+ space_line_break_pattern = r'[\s\n]+'
+ space_line_break_regex = re.compile(space_line_break_pattern)
+
+ stdout_end = stdout.decode(errors='ignore')[-1000:]
+ for question, answers in qa_patterns:
+ # first replace hard spaces by regular spaces, since they would mess up the join/split below
+ question = question.replace(r'\ ', ' ')
+ # replace spaces/line breaks with regex pattern that matches one or more spaces/line breaks,
+ # and allow extra whitespace at the end
+ question = space_line_break_pattern.join(space_line_break_regex.split(question)) + r'[\s\n]*$'
+ _log.debug(f"Checking for question pattern '{question}'...")
+ regex = re.compile(question.encode())
+ res = regex.search(stdout)
+ if res:
+ _log.debug(f"Found match for question pattern '{question}' at end of stdout: {stdout_end}")
+ # if answer is specified as a list, we take the first item as current answer,
+ # and add it to the back of the list (so we cycle through answers)
+ if isinstance(answers, list):
+ answer = answers.pop(0)
+ answers.append(answer)
+ elif isinstance(answers, str):
+ answer = answers
+ else:
+ raise EasyBuildError(f"Unknown type of answers encountered for question ({question}): {answers}")
+
+ # answer may need to be completed via pattern extracted from question
+ _log.debug(f"Raw answer for question pattern '{question}': {answer}")
+ answer = answer % {k: v.decode() for (k, v) in res.groupdict().items()}
+ answer += '\n'
+ _log.info(f"Found match for question pattern '{question}', replying with: {answer}")
+
+ try:
+ os.write(proc.stdin.fileno(), answer.encode())
+ except OSError as err:
+ raise EasyBuildError("Failed to answer question raised by interactive command: %s", err)
+
+ match_found = True
+ break
+ else:
+ _log.debug(f"No match for question pattern '{question}' at end of stdout: {stdout_end}")
+ else:
+ _log.info("No match found for question patterns, considering question wait patterns")
+ # if no match was found among question patterns,
+ # take into account patterns for non-questions (qa_wait_patterns)
+ for pattern in qa_wait_patterns:
+ # first replace hard spaces by regular spaces, since they would mess up the join/split below
+ pattern = pattern.replace(r'\ ', ' ')
+ # replace spaces/line breaks with regex pattern that matches one or more spaces/line breaks,
+ # and allow extra whitespace at the end
+ pattern = space_line_break_pattern.join(space_line_break_regex.split(pattern)) + r'[\s\n]*$'
+ regex = re.compile(pattern.encode())
+ _log.debug(f"Checking for question wait pattern '{pattern}'...")
+ if regex.search(stdout):
+ _log.info(f"Found match for question wait pattern '{pattern}'")
+ _log.debug(f"Found match for question wait pattern '{pattern}' at end of stdout: {stdout_end}")
+ match_found = True
+ break
+ else:
+ _log.debug(f"No match for question wait pattern '{pattern}' at end of stdout: {stdout_end}")
+ else:
+ _log.info("No match found for question wait patterns")
+ _log.debug(f"No match found in question (wait) patterns at end of stdout: {stdout_end}")
+
+ return match_found
+
+
@run_shell_cmd_cache
def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None,
hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True,
output_file=True, stream_output=None, asynchronous=False, task_id=None, with_hooks=True,
- qa_patterns=None, qa_wait_patterns=None):
+ qa_patterns=None, qa_wait_patterns=None, qa_timeout=100):
"""
Run specified (interactive) shell command, and capture output + exit code.
@@ -224,8 +380,9 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N
:param task_id: task ID for specified shell command (included in return value)
:param with_hooks: trigger pre/post run_shell_cmd hooks (if defined)
:param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers
- :param qa_wait_patterns: list of 2-tuples with patterns for non-questions
- and number of iterations to allow these patterns to match with end out command output
+ :param qa_wait_patterns: list of strings with patterns for non-questions
+ :param qa_timeout: amount of seconds to wait until more output is produced when there is no matching question
+
:return: Named tuple with:
- output: command output, stdout+stderr combined if split_stderr is disabled, only stdout otherwise
- exit_code: exit code of command (integer)
@@ -244,12 +401,35 @@ def to_cmd_str(cmd):
return cmd_str
- # temporarily raise a NotImplementedError until all options are implemented
- if qa_patterns or qa_wait_patterns:
- raise NotImplementedError
+ # make sure that qa_patterns is a list of 2-tuples (not a dict, or something else)
+ if qa_patterns:
+ if not isinstance(qa_patterns, list) or any(not isinstance(x, tuple) or len(x) != 2 for x in qa_patterns):
+ raise EasyBuildError("qa_patterns passed to run_shell_cmd should be a list of 2-tuples!")
+
+ interactive = bool(qa_patterns)
+
+ if qa_wait_patterns is None:
+ qa_wait_patterns = []
+
+ # keep path to current working dir in case we need to come back to it
+ try:
+ initial_work_dir = os.getcwd()
+ except FileNotFoundError:
+ raise EasyBuildError(CWD_NOTFOUND_ERROR)
if work_dir is None:
- work_dir = os.getcwd()
+ work_dir = initial_work_dir
+
+ if with_hooks:
+ hooks = load_hooks(build_option('hooks'))
+ kwargs = {
+ 'interactive': interactive,
+ 'work_dir': work_dir,
+ }
+ hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=kwargs)
+ if hook_res:
+ cmd, old_cmd = hook_res, cmd
+ _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd)
cmd_str = to_cmd_str(cmd)
@@ -263,36 +443,44 @@ def to_cmd_str(cmd):
_log.info(f"Auto-enabling streaming output of '{cmd_str}' command because logging to stdout is enabled")
stream_output = True
- # temporary output file(s) for command output
+ # temporary output file(s) for command output, along with helper scripts
if output_file:
toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output')
os.makedirs(toptmpdir, exist_ok=True)
cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0]))
tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-')
+
+ _log.info(f'run_shell_cmd: command environment of "{cmd_str}" will be saved to {tmpdir}')
+
cmd_out_fp = os.path.join(tmpdir, 'out.txt')
- _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}')
+ _log.info(f'run_shell_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}')
if split_stderr:
cmd_err_fp = os.path.join(tmpdir, 'err.txt')
- _log.info(f'run_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}')
+ _log.info(f'run_shell_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}')
else:
cmd_err_fp = None
+
+ cmd_sh = create_cmd_scripts(cmd_str, work_dir, env, tmpdir, cmd_out_fp, cmd_err_fp)
else:
- cmd_out_fp, cmd_err_fp = None, None
+ tmpdir, cmd_out_fp, cmd_err_fp, cmd_sh = None, None, None, None
+
+ interactive_msg = 'interactive ' if interactive else ''
# early exit in 'dry run' mode, after printing the command that would be run (unless 'hidden' is enabled)
if not in_dry_run and build_option('extended_dry_run'):
if not hidden or verbose_dry_run:
silent = build_option('silent')
- msg = f" running shell command \"{cmd_str}\"\n"
+ msg = f" running {interactive_msg}shell command \"{cmd_str}\"\n"
msg += f" (in {work_dir})"
dry_run_msg(msg, silent=silent)
return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir,
- out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id, task_id=task_id)
+ out_file=cmd_out_fp, err_file=cmd_err_fp, cmd_sh=cmd_sh,
+ thread_id=thread_id, task_id=task_id)
start_time = datetime.now()
if not hidden:
- _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id)
+ _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, tmpdir, thread_id, interactive=interactive)
if stream_output:
print_msg(f"(streaming) output for command '{cmd_str}':")
@@ -307,17 +495,9 @@ def to_cmd_str(cmd):
else:
executable, shell = None, False
- if with_hooks:
- hooks = load_hooks(build_option('hooks'))
- hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': work_dir})
- if hook_res:
- cmd, old_cmd = hook_res, cmd
- cmd_str = to_cmd_str(cmd)
- _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd)
-
stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT
- log_msg = f"Running shell command '{cmd_str}' in {work_dir}"
+ log_msg = f"Running {interactive_msg}shell command '{cmd_str}' in {work_dir}"
if thread_id:
log_msg += f" (via thread with ID {thread_id})"
_log.info(log_msg)
@@ -329,29 +509,90 @@ def to_cmd_str(cmd):
if stdin:
stdin = stdin.encode()
- if stream_output:
+ if stream_output or qa_patterns:
+
+ if qa_patterns:
+ # make stdout, stderr, stdin non-blocking files
+ channels = [proc.stdout, proc.stdin]
+ if split_stderr:
+ channels += proc.stderr
+ for channel in channels:
+ fd = channel.fileno()
+ flags = fcntl.fcntl(fd, fcntl.F_GETFL)
+ fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
if stdin:
proc.stdin.write(stdin)
+ proc.stdin.flush()
+ if not qa_patterns:
+ proc.stdin.close()
exit_code = None
stdout, stderr = b'', b''
+ check_interval_secs = 0.1
+ time_no_match = 0
+ prev_stdout = ''
+ # collect output piece-wise, while checking for questions to answer (if qa_patterns is provided)
while exit_code is None:
- exit_code = proc.poll()
# use small read size (128 bytes) when streaming output, to make it stream more fluently
# -1 means reading until EOF
read_size = 128 if exit_code is None else -1
- stdout += proc.stdout.read(read_size)
+ # get output as long as output is available;
+ # note: can't use proc.stdout.read without read_size argument,
+ # since that will always wait until EOF
+ more_stdout = True
+ while more_stdout:
+ more_stdout = proc.stdout.read(read_size) or b''
+ _log.debug(f"Obtained more stdout: {more_stdout}")
+ stdout += more_stdout
+
+ # note: we assume that there won't be any questions in stderr output
if split_stderr:
- stderr += proc.stderr.read(read_size)
+ more_stderr = True
+ while more_stderr:
+ more_stderr = proc.stderr.read(read_size) or b''
+ stderr += more_stderr
+
+ if qa_patterns:
+ # only check for question patterns if additional output is available
+ # compared to last time a question was answered;
+ # use empty list of question patterns if no extra output (except for whitespace) is available
+ # we do always need to check for wait patterns though!
+ active_qa_patterns = qa_patterns if stdout.strip() != prev_stdout else []
+
+ if _answer_question(stdout, proc, active_qa_patterns, qa_wait_patterns):
+ time_no_match = 0
+ prev_stdout = stdout.strip()
+ else:
+ # this will only run if the for loop above was *not* stopped by the break statement
+ time_no_match += check_interval_secs
+ if time_no_match > qa_timeout:
+ error_msg = "No matching questions found for current command output, "
+ error_msg += f"giving up after {qa_timeout} seconds!"
+ raise EasyBuildError(error_msg)
+ else:
+ _log.debug(f"{time_no_match:0.1f} seconds without match in output of interactive shell command")
+
+ time.sleep(check_interval_secs)
+
+ exit_code = proc.poll()
+
+ # collect last bit of output once processed has exited
+ stdout += proc.stdout.read()
+ if split_stderr:
+ stderr += proc.stderr.read()
else:
(stdout, stderr) = proc.communicate(input=stdin)
# return output as a regular string rather than a byte sequence (and non-UTF-8 characters get stripped out)
- output = stdout.decode('utf-8', 'ignore')
- stderr = stderr.decode('utf-8', 'ignore') if split_stderr else None
+ # getpreferredencoding normally gives 'utf-8' but can be ASCII (ANSI_X3.4-1968)
+ # for Python 3.6 and older with LC_ALL=C
+ encoding = locale.getpreferredencoding(False)
+ output = stdout.decode(encoding, 'ignore')
+ stderr = stderr.decode(encoding, 'ignore') if split_stderr else None
# store command output to temporary file(s)
if output_file:
@@ -364,8 +605,9 @@ def to_cmd_str(cmd):
except IOError as err:
raise EasyBuildError(f"Failed to dump command output to temporary file: {err}")
- res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir,
- out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id, task_id=task_id)
+ res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr,
+ work_dir=work_dir, out_file=cmd_out_fp, err_file=cmd_err_fp, cmd_sh=cmd_sh,
+ thread_id=thread_id, task_id=task_id)
# always log command output
cmd_name = cmd_str.split(' ')[0]
@@ -375,16 +617,31 @@ def to_cmd_str(cmd):
else:
_log.info(f"Output of '{cmd_name} ...' shell command (stdout + stderr):\n{res.output}")
- if res.exit_code == 0:
+ if res.exit_code == EasyBuildExit.SUCCESS:
_log.info(f"Shell command completed successfully (see output above): {cmd_str}")
else:
_log.warning(f"Shell command FAILED (exit code {res.exit_code}, see output above): {cmd_str}")
if fail_on_error:
raise_run_shell_cmd_error(res)
+ # check that we still are in a sane environment after command execution
+ # safeguard against commands that deleted the work dir or missbehaving filesystems
+ try:
+ os.getcwd()
+ except FileNotFoundError:
+ _log.warning(
+ f"Shell command `{cmd_str}` completed successfully but left the system in an unknown working directory. "
+ f"Changing back to initial working directory: {initial_work_dir}"
+ )
+ try:
+ os.chdir(initial_work_dir)
+ except OSError as err:
+ raise EasyBuildError(f"Failed to return to {initial_work_dir} after executing command `{cmd_str}`: {err}")
+
if with_hooks:
run_hook_kwargs = {
'exit_code': res.exit_code,
+ 'interactive': interactive,
'output': res.output,
'stderr': res.stderr,
'work_dir': res.work_dir,
@@ -398,7 +655,7 @@ def to_cmd_str(cmd):
return res
-def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id):
+def _cmd_trace_msg(cmd, start_time, work_dir, stdin, tmpdir, thread_id, interactive=False):
"""
Helper function to construct and print trace message for command being run
@@ -406,16 +663,17 @@ def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thr
:param start_time: datetime object indicating when command was started
:param work_dir: path of working directory in which command is run
:param stdin: stdin input value for command
- :param cmd_out_fp: path to output file for command
- :param cmd_err_fp: path to errors/warnings output file for command
+ :param tmpdir: path to temporary output directory for command
:param thread_id: thread ID (None when not running shell command asynchronously)
+ :param interactive: boolean indicating whether it is an interactive command, or not
"""
start_time = start_time.strftime('%Y-%m-%d %H:%M:%S')
+ interactive = 'interactive ' if interactive else ''
if thread_id:
- run_cmd_msg = f"running shell command (asynchronously, thread ID: {thread_id}):"
+ run_cmd_msg = f"running {interactive}shell command (asynchronously, thread ID: {thread_id}):"
else:
- run_cmd_msg = "running shell command:"
+ run_cmd_msg = f"running {interactive}shell command:"
lines = [
run_cmd_msg,
@@ -425,725 +683,12 @@ def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thr
]
if stdin:
lines.append(f"\t[input: {stdin}]")
- if cmd_out_fp:
- lines.append(f"\t[output saved to {cmd_out_fp}]")
- if cmd_err_fp:
- lines.append(f"\t[errors/warnings saved to {cmd_err_fp}]")
+ if tmpdir:
+ lines.append(f"\t[output and state saved to {tmpdir}]")
trace_msg('\n'.join(lines))
-def get_output_from_process(proc, read_size=None, asynchronous=False):
- """
- Get output from running process (that was opened with subprocess.Popen).
-
- :param proc: process to get output from
- :param read_size: number of bytes of output to read (if None: read all output)
- :param asynchronous: get output asynchronously
- """
-
- if asynchronous:
- # e=False is set to avoid raising an exception when command has completed;
- # that's needed to ensure we get all output,
- # see https://github.com/easybuilders/easybuild-framework/issues/3593
- output = asyncprocess.recv_some(proc, e=False)
- elif read_size:
- output = proc.stdout.read(read_size)
- else:
- output = proc.stdout.read()
-
- # need to be careful w.r.t. encoding since we want to obtain a string value,
- # and the output may include non UTF-8 characters
- # * in Python 2, .decode() returns a value of type 'unicode',
- # but we really want a regular 'str' value (which is also why we use 'ignore' for encoding errors)
- # * in Python 3, .decode() returns a 'str' value when called on the 'bytes' value obtained from .read()
- output = str(output.decode('ascii', 'ignore'))
-
- return output
-
-
-@run_cmd_cache
-def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None,
- force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False,
- with_hooks=True):
- """
- Run specified command (in a subshell)
- :param cmd: command to run
- :param log_ok: only run output/exit code for failing commands (exit code non-zero)
- :param log_all: always log command output and exit code
- :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
- :param inp: the input given to the command via stdin
- :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
- :param log_output: indicate whether all output of command should be logged to a separate temporary logfile
- :param path: path to execute the command in; current working directory is used if unspecified
- :param force_in_dry_run: force running the command during dry run
- :param verbose: include message on running the command in dry run output
- :param shell: allow commands to not run in a shell (especially useful for cmd lists), defaults to True
- :param trace: print command being executed as part of trace output
- :param stream_output: enable streaming command output to stdout
- :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True)
- :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined)
- """
- cwd = os.getcwd()
-
- if isinstance(cmd, str):
- cmd_msg = cmd.strip()
- elif isinstance(cmd, list):
- cmd_msg = ' '.join(cmd)
- else:
- raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd)
-
- if shell is None:
- shell = True
- if isinstance(cmd, list):
- raise EasyBuildError("When passing cmd as a list then `shell` must be set explictely! "
- "Note that all elements of the list but the first are treated as arguments "
- "to the shell and NOT to the command to be executed!")
-
- if log_output or (trace and build_option('trace')):
- # collect output of running command in temporary log file, if desired
- fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-')
- os.close(fd)
- try:
- cmd_log = open(cmd_log_fn, 'w')
- except IOError as err:
- raise EasyBuildError("Failed to open temporary log file for output of command: %s", err)
- _log.debug('run_cmd: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
- else:
- cmd_log_fn, cmd_log = None, None
-
- # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely
- if stream_output is None and build_option('logtostdout'):
- _log.info("Auto-enabling streaming output of '%s' command because logging to stdout is enabled", cmd_msg)
- stream_output = True
-
- if stream_output:
- print_msg("(streaming) output for command '%s':" % cmd_msg)
-
- start_time = datetime.now()
- if trace:
- trace_txt = "running command:\n"
- trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
- trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
- if inp:
- trace_txt += "\t[input: %s]\n" % inp
- trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
- trace_msg(trace_txt + '\t' + cmd_msg)
-
- # early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced)
- if not force_in_dry_run and build_option('extended_dry_run'):
- if path is None:
- path = cwd
- if verbose:
- dry_run_msg(" running command \"%s\"" % cmd_msg, silent=build_option('silent'))
- dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
-
- # make sure we get the type of the return value right
- if simple:
- return True
- else:
- # output, exit code
- return ('', 0)
-
- try:
- if path:
- os.chdir(path)
-
- _log.debug("run_cmd: running cmd %s (in %s)" % (cmd, os.getcwd()))
- except OSError as err:
- _log.warning("Failed to change to %s: %s" % (path, err))
- _log.info("running cmd %s in non-existing directory, might fail!", cmd)
-
- if cmd_log:
- cmd_log.write("# output for command: %s\n\n" % cmd_msg)
-
- exec_cmd = "/bin/bash"
-
- if not shell:
- if isinstance(cmd, list):
- exec_cmd = None
- cmd.insert(0, '/usr/bin/env')
- elif isinstance(cmd, str):
- cmd = '/usr/bin/env %s' % cmd
- else:
- raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd))
-
- if with_hooks:
- hooks = load_hooks(build_option('hooks'))
- hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()})
- if isinstance(hook_res, str):
- cmd, old_cmd = hook_res, cmd
- _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd)
-
- _log.info('running cmd: %s ' % cmd)
- try:
- proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
- stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd)
- except OSError as err:
- raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err)
-
- if inp:
- proc.stdin.write(inp.encode())
- proc.stdin.close()
-
- if asynchronous:
- return (proc, cmd, cwd, start_time, cmd_log)
- else:
- return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple,
- regexp=regexp, stream_output=stream_output, trace=trace, with_hook=with_hooks)
-
-
-def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''):
- """
- Check status of command that was started asynchronously.
-
- :param proc: subprocess.Popen instance representing asynchronous command
- :param cmd: command being run
- :param owd: original working directory
- :param start_time: start time of command (datetime instance)
- :param cmd_log: log file to print command output to
- :param fail_on_error: raise EasyBuildError when command exited with an error
- :param output_read_size: number of bytes to read from output
- :param output: already collected output for this command
-
- :result: dict value with result of the check (boolean 'done', 'exit_code', 'output')
- """
- # use small read size, to avoid waiting for a long time until sufficient output is produced
- if output_read_size:
- if not isinstance(output_read_size, int) or output_read_size < 0:
- raise EasyBuildError("Number of output bytes to read should be a positive integer value (or zero)")
- add_out = get_output_from_process(proc, read_size=output_read_size)
- _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out))
- output += add_out
-
- exit_code = proc.poll()
- if exit_code is None:
- _log.debug("Asynchronous command '%s' still running..." % cmd)
- done = False
- else:
- _log.debug("Asynchronous command '%s' completed!", cmd)
- output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output,
- simple=False, trace=False, log_ok=fail_on_error)
- done = True
-
- res = {
- 'done': done,
- 'exit_code': exit_code,
- 'output': output,
- }
- return res
-
-
-def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False,
- regexp=True, stream_output=None, trace=True, output='', with_hook=True):
- """
- Complete running of command represented by passed subprocess.Popen instance.
-
- :param proc: subprocess.Popen instance representing running command
- :param cmd: command being run
- :param owd: original working directory
- :param start_time: start time of command (datetime instance)
- :param cmd_log: log file to print command output to
- :param log_ok: only run output/exit code for failing commands (exit code non-zero)
- :param log_all: always log command output and exit code
- :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
- :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
- :param stream_output: enable streaming command output to stdout
- :param trace: print command being executed as part of trace output
- :param with_hook: trigger post run_shell_cmd hooks (if defined)
- """
- # use small read size when streaming output, to make it stream more fluently
- # read size should not be too small though, to avoid too much overhead
- if stream_output:
- read_size = 128
- else:
- read_size = 1024 * 8
-
- stdouterr = output
-
- try:
- ec = proc.poll()
- while ec is None:
- # need to read from time to time.
- # - otherwise the stdout/stderr buffer gets filled and it all stops working
- output = get_output_from_process(proc, read_size=read_size)
- if cmd_log:
- cmd_log.write(output)
- if stream_output:
- sys.stdout.write(output)
- stdouterr += output
- ec = proc.poll()
-
- # read remaining data (all of it)
- output = get_output_from_process(proc)
- finally:
- proc.stdout.close()
-
- if cmd_log:
- cmd_log.write(output)
- cmd_log.close()
- if stream_output:
- sys.stdout.write(output)
- stdouterr += output
-
- if with_hook:
- hooks = load_hooks(build_option('hooks'))
- run_hook_kwargs = {
- 'exit_code': ec,
- 'output': stdouterr,
- 'work_dir': os.getcwd(),
- }
- run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
-
- if trace:
- trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
-
- try:
- os.chdir(owd)
- except OSError as err:
- raise EasyBuildError("Failed to return to %s after executing command: %s", owd, err)
-
- return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp)
-
-
-def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None,
- maxhits=50, trace=True):
- """
- Run specified interactive command (in a subshell)
- :param cmd: command to run
- :param qa: dictionary which maps question to answers
- :param no_qa: list of patters that are not questions
- :param log_ok: only run output/exit code for failing commands (exit code non-zero)
- :param log_all: always log command output and exit code
- :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
- :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
- :param std_qa: dictionary which maps question regex patterns to answers
- :param path: path to execute the command is; current working directory is used if unspecified
- :param maxhits: maximum number of cycles (seconds) without being able to find a known question
- :param trace: print command being executed as part of trace output
- """
- cwd = os.getcwd()
-
- if not isinstance(cmd, str) and len(cmd) > 1:
- # We use shell=True and hence we should really pass the command as a string
- # When using a list then every element past the first is passed to the shell itself, not the command!
- raise EasyBuildError("The command passed must be a string!")
-
- if log_all or (trace and build_option('trace')):
- # collect output of running command in temporary log file, if desired
- fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-')
- os.close(fd)
- try:
- cmd_log = open(cmd_log_fn, 'w')
- except IOError as err:
- raise EasyBuildError("Failed to open temporary log file for output of interactive command: %s", err)
- _log.debug('run_cmd_qa: Output of "%s" will be logged to %s' % (cmd, cmd_log_fn))
- else:
- cmd_log_fn, cmd_log = None, None
-
- start_time = datetime.now()
- if trace:
- trace_txt = "running interactive command:\n"
- trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S')
- trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd())
- trace_txt += "\t[output logged in %s]\n" % cmd_log_fn
- trace_msg(trace_txt + '\t' + cmd.strip())
-
- # early exit in 'dry run' mode, after printing the command that would be run
- if build_option('extended_dry_run'):
- if path is None:
- path = cwd
- dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent'))
- dry_run_msg(" (in %s)" % path, silent=build_option('silent'))
- if cmd_log:
- cmd_log.close()
- if simple:
- return True
- else:
- # output, exit code
- return ('', 0)
-
- try:
- if path:
- os.chdir(path)
-
- _log.debug("run_cmd_qa: running cmd %s (in %s)" % (cmd, os.getcwd()))
- except OSError as err:
- _log.warning("Failed to change to %s: %s" % (path, err))
- _log.info("running cmd %s in non-existing directory, might fail!" % cmd)
-
- # Part 1: process the QandA dictionary
- # given initial set of Q and A (in dict), return dict of reg. exp. and A
- #
- # make regular expression that matches the string with
- # - replace whitespace
- # - replace newline
-
- def escape_special(string):
- return re.sub(r"([\+\?\(\)\[\]\*\.\\\$])", r"\\\1", string)
-
- split = r'[\s\n]+'
- regSplit = re.compile(r"" + split)
-
- def process_QA(q, a_s):
- splitq = [escape_special(x) for x in regSplit.split(q)]
- regQtxt = split.join(splitq) + split.rstrip('+') + "*$"
- # add optional split at the end
- for i in [idx for idx, a in enumerate(a_s) if not a.endswith('\n')]:
- a_s[i] += '\n'
- regQ = re.compile(r"" + regQtxt)
- if regQ.search(q):
- return (a_s, regQ)
- else:
- raise EasyBuildError("runqanda: Question %s converted in %s does not match itself", q, regQtxt)
-
- def check_answers_list(answers):
- """Make sure we have a list of answers (as strings)."""
- if isinstance(answers, str):
- answers = [answers]
- elif not isinstance(answers, list):
- if cmd_log:
- cmd_log.close()
- raise EasyBuildError("Invalid type for answer on %s, no string or list: %s (%s)",
- question, type(answers), answers)
- # list is manipulated when answering matching question, so return a copy
- return answers[:]
-
- new_qa = {}
- _log.debug("new_qa: ")
- for question, answers in qa.items():
- answers = check_answers_list(answers)
- (answers, regQ) = process_QA(question, answers)
- new_qa[regQ] = answers
- _log.debug("new_qa[%s]: %s" % (regQ.pattern, new_qa[regQ]))
-
- new_std_qa = {}
- if std_qa:
- for question, answers in std_qa.items():
- regQ = re.compile(r"" + question + r"[\s\n]*$")
- answers = check_answers_list(answers)
- for i in [idx for idx, a in enumerate(answers) if not a.endswith('\n')]:
- answers[i] += '\n'
- new_std_qa[regQ] = answers
- _log.debug("new_std_qa[%s]: %s" % (regQ.pattern, new_std_qa[regQ]))
-
- new_no_qa = []
- if no_qa:
- # simple statements, can contain wildcards
- new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa]
-
- _log.debug("New noQandA list is: %s" % [x.pattern for x in new_no_qa])
-
- # Part 2: Run the command and answer questions
- # - this needs asynchronous stdout
-
- hooks = load_hooks(build_option('hooks'))
- run_hook_kwargs = {
- 'interactive': True,
- 'work_dir': os.getcwd(),
- }
- hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
- if isinstance(hook_res, str):
- cmd, old_cmd = hook_res, cmd
- _log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')",
- RUN_SHELL_CMD, cmd, old_cmd)
-
- # # Log command output
- if cmd_log:
- cmd_log.write("# output for interactive command: %s\n\n" % cmd)
-
- # Make sure we close the proc handles and the cmd_log file
- @contextlib.contextmanager
- def get_proc():
- try:
- proc = asyncprocess.Popen(cmd, shell=True, stdout=asyncprocess.PIPE, stderr=asyncprocess.STDOUT,
- stdin=asyncprocess.PIPE, close_fds=True, executable='/bin/bash')
- except OSError as err:
- if cmd_log:
- cmd_log.close()
- raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err)
- try:
- yield proc
- finally:
- if proc.stdout:
- proc.stdout.close()
- if proc.stdin:
- proc.stdin.close()
- if cmd_log:
- cmd_log.close()
-
- with get_proc() as proc:
- ec = proc.poll()
- stdout_err = ''
- old_len_out = -1
- hit_count = 0
-
- while ec is None:
- # need to read from time to time.
- # - otherwise the stdout/stderr buffer gets filled and it all stops working
- try:
- out = get_output_from_process(proc, asynchronous=True)
-
- if cmd_log:
- cmd_log.write(out)
- stdout_err += out
- # recv_some used by get_output_from_process for getting asynchronous output may throw exception
- except (IOError, Exception) as err:
- _log.debug("run_cmd_qa cmd %s: read failed: %s", cmd, err)
- out = None
-
- hit = False
- for question, answers in new_qa.items():
- res = question.search(stdout_err)
- if out and res:
- fa = answers[0] % res.groupdict()
- # cycle through list of answers
- last_answer = answers.pop(0)
- answers.append(last_answer)
- _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers)
-
- _log.debug("run_cmd_qa answer %s question %s out %s", fa, question.pattern, stdout_err[-50:])
- asyncprocess.send_all(proc, fa)
- hit = True
- break
- if not hit:
- for question, answers in new_std_qa.items():
- res = question.search(stdout_err)
- if out and res:
- fa = answers[0] % res.groupdict()
- # cycle through list of answers
- last_answer = answers.pop(0)
- answers.append(last_answer)
- _log.debug("List of answers for question %s after cycling: %s", question.pattern, answers)
-
- _log.debug("run_cmd_qa answer %s std question %s out %s",
- fa, question.pattern, stdout_err[-50:])
- asyncprocess.send_all(proc, fa)
- hit = True
- break
- if not hit:
- if len(stdout_err) > old_len_out:
- old_len_out = len(stdout_err)
- else:
- noqa = False
- for r in new_no_qa:
- if r.search(stdout_err):
- _log.debug("runqanda: noQandA found for out %s", stdout_err[-50:])
- noqa = True
- if not noqa:
- hit_count += 1
- else:
- hit_count = 0
- else:
- hit_count = 0
-
- if hit_count > maxhits:
- # explicitly kill the child process before exiting
- try:
- os.killpg(proc.pid, signal.SIGKILL)
- os.kill(proc.pid, signal.SIGKILL)
- except OSError as err:
- _log.debug("run_cmd_qa exception caught when killing child process: %s", err)
- _log.debug("run_cmd_qa: full stdouterr: %s", stdout_err)
- raise EasyBuildError("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s",
- cmd, maxhits, stdout_err[-500:])
-
- # the sleep below is required to avoid exiting on unknown 'questions' too early (see above)
- time.sleep(1)
- ec = proc.poll()
-
- # Process stopped. Read all remaining data
- try:
- if proc.stdout:
- out = get_output_from_process(proc)
- stdout_err += out
- if cmd_log:
- cmd_log.write(out)
- except IOError as err:
- _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err)
-
- run_hook_kwargs.update({
- 'interactive': True,
- 'exit_code': ec,
- 'output': stdout_err,
- })
- run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
-
- if trace:
- trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
-
- try:
- os.chdir(cwd)
- except OSError as err:
- raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err)
-
- return parse_cmd_output(cmd, stdout_err, ec, simple, log_all, log_ok, regexp)
-
-
-def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp):
- """
- Parse command output and construct return value.
- :param cmd: executed command
- :param stdouterr: combined stdout/stderr of executed command
- :param ec: exit code of executed command
- :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code)
- :param log_all: always log command output and exit code
- :param log_ok: only run output/exit code for failing commands (exit code non-zero)
- :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
- """
- if strictness == IGNORE:
- check_ec = False
- fail_on_error_match = False
- elif strictness == WARN:
- check_ec = True
- fail_on_error_match = False
- elif strictness == ERROR:
- check_ec = True
- fail_on_error_match = True
- else:
- raise EasyBuildError("invalid strictness setting: %s", strictness)
-
- # allow for overriding the regexp setting
- if not regexp:
- fail_on_error_match = False
-
- if ec and (log_all or log_ok):
- # We don't want to error if the user doesn't care
- if check_ec:
- raise EasyBuildError('cmd "%s" exited with exit code %s and output:\n%s', cmd, ec, stdouterr)
- else:
- _log.warning('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
- elif not ec:
- if log_all:
- _log.info('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
- else:
- _log.debug('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
-
- # parse the stdout/stderr for errors when strictness dictates this or when regexp is passed in
- if fail_on_error_match or regexp:
- res = parse_log_for_error(stdouterr, regexp, stdout=False)
- if res:
- errors = "\n\t" + "\n\t".join([r[0] for r in res])
- error_str = "error" if len(res) == 1 else "errors"
- if fail_on_error_match:
- raise EasyBuildError("Found %s %s in output of %s:%s", len(res), error_str, cmd, errors)
- else:
- _log.warning("Found %s potential %s (some may be harmless) in output of %s:%s",
- len(res), error_str, cmd, errors)
-
- if simple:
- if ec:
- # If the user does not care -> will return true
- return not check_ec
- else:
- return True
- else:
- # Because we are not running in simple mode, we return the output and ec to the user
- return (stdouterr, ec)
-
-
-def parse_log_for_error(txt, regExp=None, stdout=True, msg=None):
- """
- txt is multiline string.
- - in memory
- regExp is a one-line regular expression
- - default
- """
- global errors_found_in_log
-
- if regExp and isinstance(regExp, bool):
- regExp = r"(? 0)
+ found = (res.exit_code == EasyBuildExit.SUCCESS and int(res.output.strip()) > 0)
except ValueError:
# Returned something else than an int -> Error
found = False
@@ -903,7 +901,7 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False):
"""
res = run_shell_cmd(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True,
hidden=True, with_hooks=False, output_file=False, stream_output=False)
- if not ignore_ec and res.exit_code:
+ if not ignore_ec and res.exit_code != EasyBuildExit.SUCCESS:
_log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, res.output))
return UNKNOWN
else:
@@ -917,7 +915,7 @@ def get_gcc_version():
res = run_shell_cmd('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True,
output_file=False, stream_output=False)
gcc_ver = None
- if res.exit_code:
+ if res.exit_code != EasyBuildExit.SUCCESS:
_log.warning("Failed to determine the version of GCC: %s", res.output)
gcc_ver = UNKNOWN
@@ -972,7 +970,7 @@ def get_linked_libs_raw(path):
"""
res = run_shell_cmd("file %s" % path, fail_on_error=False, hidden=True, output_file=False, stream_output=False)
- if res.exit_code:
+ if res.exit_code != EasyBuildExit.SUCCESS:
fail_msg = "Failed to run 'file %s': %s" % (path, res.output)
_log.warning(fail_msg)
@@ -1007,7 +1005,7 @@ def get_linked_libs_raw(path):
# like printing 'not a dynamic executable' when not enough memory is available
# (see also https://bugzilla.redhat.com/show_bug.cgi?id=1817111)
res = run_shell_cmd(linked_libs_cmd, fail_on_error=False, hidden=True, output_file=False, stream_output=False)
- if res.exit_code == 0:
+ if res.exit_code == EasyBuildExit.SUCCESS:
linked_libs_out = res.output
else:
fail_msg = "Determining linked libraries for %s via '%s' failed! Output: '%s'"
@@ -1195,8 +1193,9 @@ def get_default_parallelism():
else:
maxuserproc = int(res.output)
except ValueError as err:
- raise EasyBuildError("Failed to determine max user processes (%s, %s): %s",
- res.exit_code, res.output, err)
+ raise EasyBuildError(
+ "Failed to determine max user processes (%s, %s): %s", res.exit_code, res.output, err
+ )
# assume 6 processes per build thread + 15 overhead
par_guess = (maxuserproc - 15) // 6
if par_guess < par:
@@ -1367,9 +1366,9 @@ def extract_version(tool):
python_version = extract_version(sys.executable)
opt_dep_versions = {}
- for key in EASYBUILD_OPTIONAL_DEPENDENCIES:
+ for key, opt_dep in EASYBUILD_OPTIONAL_DEPENDENCIES.items():
- pkg = EASYBUILD_OPTIONAL_DEPENDENCIES[key][0]
+ pkg = opt_dep[0]
if pkg is None:
pkg = key.lower()
@@ -1395,8 +1394,8 @@ def extract_version(tool):
opt_deps_key = "Optional dependencies"
checks_data[opt_deps_key] = {}
- for key in opt_dep_versions:
- checks_data[opt_deps_key][key] = (opt_dep_versions[key], EASYBUILD_OPTIONAL_DEPENDENCIES[key][1])
+ for key, version in opt_dep_versions.items():
+ checks_data[opt_deps_key][key] = (version, EASYBUILD_OPTIONAL_DEPENDENCIES[key][1])
sys_tools_key = "System tools"
checks_data[sys_tools_key] = {}
diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py
index ffd8ce580b..36d87f0090 100644
--- a/easybuild/tools/testing.py
+++ b/easybuild/tools/testing.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -47,7 +47,7 @@
from easybuild.framework.easyconfig.tools import skip_available
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option
-from easybuild.tools.filetools import find_easyconfigs, mkdir, read_file, write_file
+from easybuild.tools.filetools import find_easyconfigs, get_cwd, mkdir, read_file, write_file
from easybuild.tools.github import GITHUB_EASYBLOCKS_REPO, GITHUB_EASYCONFIGS_REPO, create_gist, post_comment_in_issue
from easybuild.tools.jenkins import aggregate_xml_in_dirs
from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel
@@ -67,7 +67,7 @@ def regtest(easyconfig_paths, modtool, build_specs=None):
:param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
- cur_dir = os.getcwd()
+ cur_dir = get_cwd()
aggregate_regtest = build_option('aggregate_regtest')
if aggregate_regtest is not None:
@@ -321,8 +321,8 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state,
gpu_info = get_gpu_info()
gpu_str = ""
if gpu_info:
- for vendor in gpu_info:
- for gpu, num in gpu_info[vendor].items():
+ for vendor, vendor_gpu in gpu_info.items():
+ for gpu, num in vendor_gpu.items():
gpu_str += ", %s x %s %s" % (num, vendor, gpu)
os_info = '%(hostname)s - %(os_type)s %(os_name)s %(os_version)s' % system_info
@@ -336,6 +336,14 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state,
comment_lines = ["Test report by @%s" % github_user]
+ if build_option('include_easyblocks_from_commit'):
+ if repo_type == GITHUB_EASYCONFIGS_REPO:
+ easyblocks_commit = build_option('include_easyblocks_from_commit')
+ url = 'https://github.com/%s/%s/commit/%s' % (pr_target_account, GITHUB_EASYBLOCKS_REPO, easyblocks_commit)
+ comment_lines.append("Using easyblocks from %s" % url)
+ else:
+ raise EasyBuildError("Don't know how to submit test reports to repo %s.", repo_type)
+
if build_option('include_easyblocks_from_pr'):
if repo_type == GITHUB_EASYCONFIGS_REPO:
easyblocks_pr_nrs = [int(x) for x in build_option('include_easyblocks_from_pr')]
diff --git a/easybuild/tools/toolchain/__init__.py b/easybuild/tools/toolchain/__init__.py
index 876d16c44f..cf755a8e40 100644
--- a/easybuild/tools/toolchain/__init__.py
+++ b/easybuild/tools/toolchain/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py
index 6387273571..de69214388 100644
--- a/easybuild/tools/toolchain/compiler.py
+++ b/easybuild/tools/toolchain/compiler.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -314,7 +314,7 @@ def _set_optimal_architecture(self, default_optarch=None):
optarch = build_option('optarch')
# --optarch is specified with flags to use
- if optarch is not None and isinstance(optarch, dict):
+ if isinstance(optarch, dict):
# optarch has been validated as complex string with multiple compilers and converted to a dictionary
# first try module names, then the family in optarch
current_compiler_names = (getattr(self, 'COMPILER_MODULE_NAME', []) +
@@ -329,14 +329,12 @@ def _set_optimal_architecture(self, default_optarch=None):
self.log.info("_set_optimal_architecture: no optarch found for compiler %s. Ignoring option.",
current_compiler)
- use_generic = False
- if optarch is not None:
- # optarch has been parsed as a simple string
- if isinstance(optarch, str):
- if optarch == OPTARCH_GENERIC:
- use_generic = True
- else:
- raise EasyBuildError("optarch is neither an string or a dict %s. This should never happen", optarch)
+ if isinstance(optarch, str):
+ use_generic = (optarch == OPTARCH_GENERIC)
+ elif optarch is None:
+ use_generic = False
+ else:
+ raise EasyBuildError("optarch is neither an string or a dict %s. This should never happen", optarch)
if use_generic:
if (self.arch, self.cpu_family) in (self.COMPILER_GENERIC_OPTION or []):
@@ -351,16 +349,17 @@ def _set_optimal_architecture(self, default_optarch=None):
optarch = self.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(self.arch, self.cpu_family)]
if optarch is not None:
- self.log.info("_set_optimal_architecture: using %s as optarch for %s.", optarch, self.arch)
+ optarch_log_str = optarch or 'no flags'
+ self.log.info("_set_optimal_architecture: using %s as optarch for %s/%s.",
+ optarch_log_str, self.arch, self.cpu_family)
self.options.options_map['optarch'] = optarch
-
- if self.options.options_map.get('optarch', None) is None:
+ elif self.options.options_map.get('optarch', None) is None:
optarch_flags_str = "%soptarch flags" % ('', 'generic ')[use_generic]
error_msg = "Don't know how to set %s for %s/%s! " % (optarch_flags_str, self.arch, self.cpu_family)
error_msg += "Use --optarch='' to override (see "
- error_msg += "http://easybuild.readthedocs.io/en/latest/Controlling_compiler_optimization_flags.html "
+ error_msg += "https://docs.easybuild.io/controlling-compiler-optimization-flags/ "
error_msg += "for details) and consider contributing your settings back (see "
- error_msg += "http://easybuild.readthedocs.io/en/latest/Contributing.html)."
+ error_msg += "https://docs.easybuild.io/contributing/)."
raise EasyBuildError(error_msg)
def comp_family(self, prefix=None):
diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py
index 5655f71961..643e8b4d21 100644
--- a/easybuild/tools/toolchain/constants.py
+++ b/easybuild/tools/toolchain/constants.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/toolchain/fft.py b/easybuild/tools/toolchain/fft.py
index c0e1dedd1a..b7191d5a26 100644
--- a/easybuild/tools/toolchain/fft.py
+++ b/easybuild/tools/toolchain/fft.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py
index 907993571c..fd74615c4a 100644
--- a/easybuild/tools/toolchain/linalg.py
+++ b/easybuild/tools/toolchain/linalg.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py
index 841cbe8ece..1646a27ce0 100644
--- a/easybuild/tools/toolchain/mpi.py
+++ b/easybuild/tools/toolchain/mpi.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/toolchain/options.py b/easybuild/tools/toolchain/options.py
index 6b2c00c95d..fe94ba44b4 100644
--- a/easybuild/tools/toolchain/options.py
+++ b/easybuild/tools/toolchain/options.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py
index efba3d2e02..3dd76903b6 100644
--- a/easybuild/tools/toolchain/toolchain.py
+++ b/easybuild/tools/toolchain/toolchain.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -588,8 +588,8 @@ def _simulated_load_dependency_module(self, name, version, metadata, verbose=Fal
self.log.debug("Defining $EB* environment variables for software named %s", name)
env_vars = env_vars_external_module(name, version, metadata)
- for key in env_vars:
- setvar(key, env_vars[key], verbose=verbose)
+ for var, value in env_vars.items():
+ setvar(var, value, verbose=verbose)
def _load_toolchain_module(self, silent=False):
"""Load toolchain module."""
@@ -925,10 +925,11 @@ def is_rpath_wrapper(path):
"""
Check whether command at specified location already is an RPATH wrapper script rather than the actual command
"""
- in_rpath_wrappers_dir = os.path.basename(os.path.dirname(os.path.dirname(path))) == RPATH_WRAPPERS_SUBDIR
+ if os.path.basename(os.path.dirname(os.path.dirname(path))) != RPATH_WRAPPERS_SUBDIR:
+ return False
+ # Check if `rpath_args`` is called in the file
# need to use binary mode to read the file, since it may be an actual compiler command (which is a binary file)
- calls_rpath_args = b'rpath_args.py $CMD' in read_file(path, mode='rb')
- return in_rpath_wrappers_dir and calls_rpath_args
+ return b'rpath_args.py $CMD' in read_file(path, mode='rb')
def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None):
"""
@@ -1027,7 +1028,7 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
def handle_sysroot(self):
"""
- Extra stuff to be done when alternate system root is specified via --sysroot EasyBuild configuration option.
+ Extra stuff to be done when alternative system root is specified via --sysroot EasyBuild configuration option.
* Update $PKG_CONFIG_PATH to include sysroot location to pkg-config files (*.pc).
"""
diff --git a/easybuild/tools/toolchain/toolchainvariables.py b/easybuild/tools/toolchain/toolchainvariables.py
index 286b32aea3..dbb5686040 100644
--- a/easybuild/tools/toolchain/toolchainvariables.py
+++ b/easybuild/tools/toolchain/toolchainvariables.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py
index 56242b8fe4..8047778391 100644
--- a/easybuild/tools/toolchain/utilities.py
+++ b/easybuild/tools/toolchain/utilities.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py
index c201f8a6f8..482f989480 100644
--- a/easybuild/tools/toolchain/variables.py
+++ b/easybuild/tools/toolchain/variables.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py
index 1b33dd24f2..fe254ea4d2 100644
--- a/easybuild/tools/utilities.py
+++ b/easybuild/tools/utilities.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/variables.py b/easybuild/tools/variables.py
index e5fa534c45..cef771f3d8 100644
--- a/easybuild/tools/variables.py
+++ b/easybuild/tools/variables.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py
index 2ea349637d..e8cd07de4a 100644
--- a/easybuild/tools/version.py
+++ b/easybuild/tools/version.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -47,6 +47,7 @@
# This causes problems further up the dependency chain...
VERSION = LooseVersion('5.0.0.dev0')
UNKNOWN = 'UNKNOWN'
+UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS'
def get_git_revision():
@@ -87,7 +88,7 @@ def get_git_revision():
try:
from easybuild.easyblocks import VERBOSE_VERSION as EASYBLOCKS_VERSION
except Exception:
- EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS' # make sure it is smaller then anything
+ EASYBLOCKS_VERSION = UNKNOWN_EASYBLOCKS_VERSION # make sure it is smaller then anything
def this_is_easybuild():
@@ -102,3 +103,14 @@ def this_is_easybuild():
msg = msg.encode('ascii')
return msg
+
+
+def different_major_versions(v1, v2):
+ """Compare major versions"""
+ # Deal with version instances being either strings or LooseVersion
+ if isinstance(v1, str):
+ v1 = LooseVersion(v1)
+ if isinstance(v2, str):
+ v2 = LooseVersion(v2)
+
+ return v1.version[0] != v2.version[0]
diff --git a/eb b/eb
index f427c5e7e7..ca401b91c3 100755
--- a/eb
+++ b/eb
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/setup.py b/setup.py
index a05ff15c40..18d3cddc17 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -119,6 +119,8 @@ def find_rel_test():
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
"Topic :: Software Development :: Build Tools",
],
platforms="Linux",
diff --git a/test/__init__.py b/test/__init__.py
index 8793b2e4e1..20c9130dcc 100644
--- a/test/__init__.py
+++ b/test/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/__init__.py b/test/framework/__init__.py
index c6a455e689..fae73ca9e6 100644
--- a/test/framework/__init__.py
+++ b/test/framework/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/asyncprocess.py b/test/framework/asyncprocess.py
index 56666f3f50..8362e1938e 100644
--- a/test/framework/asyncprocess.py
+++ b/test/framework/asyncprocess.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/build_log.py b/test/framework/build_log.py
index dcaa1620c3..838d782a9e 100644
--- a/test/framework/build_log.py
+++ b/test/framework/build_log.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -116,11 +116,12 @@ def test_easybuildlog(self):
stderr = self.get_stderr()
self.mock_stderr(False)
- more_info = "see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html for more information"
+ more_info = "see https://docs.easybuild.io/deprecated-functionality/ for more information"
+ common_warning = "\nWARNING: Deprecated functionality, will no longer work in"
expected_stderr = '\n\n'.join([
- "\nWARNING: Deprecated functionality, will no longer work in v10000001: anotherwarning; " + more_info,
- "\nWARNING: Deprecated functionality, will no longer work in v2.0: onemorewarning",
- "\nWARNING: Deprecated functionality, will no longer work in v2.0: lastwarning",
+ common_warning + " EasyBuild v10000001: anotherwarning; " + more_info,
+ common_warning + " EasyBuild v2.0: onemorewarning",
+ common_warning + " EasyBuild v2.0: lastwarning",
]) + '\n\n'
self.assertEqual(stderr, expected_stderr)
@@ -183,7 +184,7 @@ def test_easybuildlog(self):
self.mock_stderr(False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
- "[WARNING] :: Deprecated functionality, will no longer work in v10000001: ",
+ "[WARNING] :: Deprecated functionality, will no longer work in EasyBuild v10000001: ",
"this is just a test",
"(see URLGOESHERE for more information)",
])
diff --git a/test/framework/config.py b/test/framework/config.py
index ce5eb2148e..08a1a83355 100644
--- a/test/framework/config.py
+++ b/test/framework/config.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -38,12 +38,11 @@
from unittest import TextTestRunner
import easybuild.tools.options as eboptions
-from easybuild.tools import run
from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.config import ERROR, IGNORE, WARN, BuildOptions, ConfigurationVariables
from easybuild.tools.config import build_option, build_path, get_build_log_path, get_log_filename, get_repositorypath
from easybuild.tools.config import install_path, log_file_format, log_path, source_paths
from easybuild.tools.config import update_build_option, update_build_options
-from easybuild.tools.config import BuildOptions, ConfigurationVariables
from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, init_build_options
from easybuild.tools.filetools import copy_dir, mkdir, write_file
from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX
@@ -449,7 +448,7 @@ def test_XDG_CONFIG_env_vars(self):
# $XDG_CONFIG_HOME not set, multiple directories listed in $XDG_CONFIG_DIRS
del os.environ['XDG_CONFIG_HOME'] # unset, so should become default
- os.environ['XDG_CONFIG_DIRS'] = os.pathsep.join([dir1, dir2, dir3])
+ os.environ['XDG_CONFIG_DIRS'] = os.pathsep.join([dir3, dir2, dir1])
cfg_files = [
os.path.join(dir1, 'easybuild.d', 'bar.cfg'),
os.path.join(dir1, 'easybuild.d', 'foo.cfg'),
@@ -580,9 +579,9 @@ def test_flex_robot_paths(self):
def test_strict(self):
"""Test use of --strict."""
# check default
- self.assertEqual(build_option('strict'), run.WARN)
+ self.assertEqual(build_option('strict'), WARN)
- for strict_str, strict_val in [('error', run.ERROR), ('ignore', run.IGNORE), ('warn', run.WARN)]:
+ for strict_str, strict_val in [('error', ERROR), ('ignore', IGNORE), ('warn', WARN)]:
options = init_config(args=['--strict=%s' % strict_str])
init_config(build_options={'strict': options.strict})
self.assertEqual(build_option('strict'), strict_val)
diff --git a/test/framework/containers.py b/test/framework/containers.py
index bd6dfde7f6..1d9abbf8e4 100644
--- a/test/framework/containers.py
+++ b/test/framework/containers.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2018-2023 Ghent University
+# Copyright 2018-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/docs.py b/test/framework/docs.py
index 70280892e4..3ed97ab192 100644
--- a/test/framework/docs.py
+++ b/test/framework/docs.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -59,6 +59,10 @@
|-- EB_toy_buggy
|-- ExtensionEasyBlock
| |-- DummyExtension
+| | |-- CustomDummyExtension
+| | | |-- ChildCustomDummyExtension
+| | |-- DeprecatedDummyExtension
+| | | |-- ChildDeprecatedDummyExtension
| |-- EB_toy
| | |-- EB_toy_eula
| | |-- EB_toytoy
@@ -69,10 +73,14 @@
Extension
|-- ExtensionEasyBlock
| |-- DummyExtension
+| | |-- CustomDummyExtension
+| | | |-- ChildCustomDummyExtension
+| | |-- DeprecatedDummyExtension
+| | | |-- ChildDeprecatedDummyExtension
| |-- EB_toy
| | |-- EB_toy_eula
| | |-- EB_toytoy
-| |-- Toy_Extension"""
+| |-- Toy_Extension""" # noqa
LIST_EASYBLOCKS_DETAILED_TXT = """EasyBlock (easybuild.framework.easyblock)
|-- bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py)
@@ -91,6 +99,10 @@
|-- EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py)
|-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock )
| |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py)
+| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py)
+| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py)
+| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py)
+| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py)
| |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py)
| | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py)
| | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py)
@@ -101,10 +113,14 @@
Extension (easybuild.framework.extension)
|-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock )
| |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py)
+| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py)
+| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py)
+| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py)
+| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py)
| |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py)
| | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py)
| | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py)
-| |-- Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)"""
+| |-- Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" # noqa
LIST_EASYBLOCKS_SIMPLE_RST = """* **EasyBlock**
@@ -129,6 +145,16 @@
* ExtensionEasyBlock
* DummyExtension
+
+ * CustomDummyExtension
+
+ * ChildCustomDummyExtension
+
+ * DeprecatedDummyExtension
+
+ * ChildDeprecatedDummyExtension
+
+
* EB_toy
* EB_toy_eula
@@ -145,6 +171,16 @@
* ExtensionEasyBlock
* DummyExtension
+
+ * CustomDummyExtension
+
+ * ChildCustomDummyExtension
+
+ * DeprecatedDummyExtension
+
+ * ChildDeprecatedDummyExtension
+
+
* EB_toy
* EB_toy_eula
@@ -152,7 +188,7 @@
* Toy_Extension
-"""
+""" # noqa
LIST_EASYBLOCKS_DETAILED_RST = """* **EasyBlock** (easybuild.framework.easyblock)
@@ -177,6 +213,16 @@
* ExtensionEasyBlock (easybuild.framework.extensioneasyblock )
* DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py)
+
+ * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py)
+
+ * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py)
+
+ * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py)
+
+ * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py)
+
+
* EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py)
* EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py)
@@ -193,6 +239,16 @@
* ExtensionEasyBlock (easybuild.framework.extensioneasyblock )
* DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py)
+
+ * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py)
+
+ * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py)
+
+ * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py)
+
+ * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py)
+
+
* EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py)
* EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py)
@@ -200,7 +256,7 @@
* Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)
-"""
+""" # noqa
LIST_EASYBLOCKS_SIMPLE_MD = """- **EasyBlock**
- bar
@@ -219,6 +275,10 @@
- EB_toy_buggy
- ExtensionEasyBlock
- DummyExtension
+ - CustomDummyExtension
+ - ChildCustomDummyExtension
+ - DeprecatedDummyExtension
+ - ChildDeprecatedDummyExtension
- EB_toy
- EB_toy_eula
- EB_toytoy
@@ -229,10 +289,14 @@
- **Extension**
- ExtensionEasyBlock
- DummyExtension
+ - CustomDummyExtension
+ - ChildCustomDummyExtension
+ - DeprecatedDummyExtension
+ - ChildDeprecatedDummyExtension
- EB_toy
- EB_toy_eula
- EB_toytoy
- - Toy_Extension"""
+ - Toy_Extension""" # noqa
LIST_EASYBLOCKS_DETAILED_MD = """- **EasyBlock** (easybuild.framework.easyblock)
- bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py)
@@ -251,6 +315,10 @@
- EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py)
- ExtensionEasyBlock (easybuild.framework.extensioneasyblock )
- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py)
+ - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py)
+ - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py)
+ - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py)
+ - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py)
- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py)
- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py)
- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py)
@@ -261,14 +329,18 @@
- **Extension** (easybuild.framework.extension)
- ExtensionEasyBlock (easybuild.framework.extensioneasyblock )
- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py)
+ - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py)
+ - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py)
+ - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py)
+ - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py)
- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py)
- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py)
- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py)
- - Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)"""
+ - Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" # noqa
LIST_SOFTWARE_SIMPLE_TXT = """
* GCC
-* gzip"""
+* gzip""" # noqa
GCC_DESCR = "The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, "
GCC_DESCR += "as well as libraries for these languages (libstdc++, libgcj,...)."
@@ -291,7 +363,7 @@
* gzip v1.4: GCC/4.6.3, system
* gzip v1.5: foss/2018a, intel/2018a
-""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa
LIST_SOFTWARE_SIMPLE_RST = """List of supported software
==========================
@@ -307,7 +379,7 @@
---
* GCC
-* gzip"""
+* gzip""" # noqa
LIST_SOFTWARE_DETAILED_RST = """List of supported software
==========================
@@ -357,7 +429,7 @@
``1.4`` ``GCC/4.6.3``, ``system``
``1.5`` ``foss/2018a``, ``intel/2018a``
======= ===============================
-""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa
LIST_SOFTWARE_SIMPLE_MD = """# List of supported software
@@ -369,7 +441,7 @@
## G
* GCC
-* gzip"""
+* gzip""" # noqa
LIST_SOFTWARE_DETAILED_MD = """# List of supported software
@@ -403,7 +475,7 @@
version|toolchain
-------|-------------------------------
``1.4``|``GCC/4.6.3``, ``system``
-``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa
LIST_SOFTWARE_SIMPLE_MD = """# List of supported software
@@ -415,7 +487,7 @@
## G
* GCC
-* gzip"""
+* gzip""" # noqa
LIST_SOFTWARE_DETAILED_MD = """# List of supported software
@@ -449,7 +521,7 @@
version|toolchain
-------|-------------------------------
``1.4``|``GCC/4.6.3``, ``system``
-``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa
LIST_SOFTWARE_SIMPLE_JSON = """[
{
@@ -458,7 +530,7 @@
{
"name": "gzip"
}
-]"""
+]""" # noqa
LIST_SOFTWARE_DETAILED_JSON = """[
{
@@ -501,7 +573,7 @@
"version": "1.5",
"versionsuffix": ""
}
-]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa
class DocsTest(EnhancedTestCase):
@@ -513,8 +585,20 @@ def test_get_easyblock_classes(self):
# result should correspond with test easyblocks in test/framework/sandbox/easybuild/easyblocks/generic
eb_classes = get_easyblock_classes('easybuild.easyblocks.generic')
eb_names = [x.__name__ for x in eb_classes]
- expected = ['ConfigureMake', 'DummyExtension', 'MakeCp', 'ModuleRC',
- 'PythonBundle', 'Toolchain', 'Toy_Extension', 'bar']
+ expected = [
+ 'ChildCustomDummyExtension',
+ 'ChildDeprecatedDummyExtension',
+ 'ConfigureMake',
+ 'CustomDummyExtension',
+ 'DeprecatedDummyExtension',
+ 'DummyExtension',
+ 'MakeCp',
+ 'ModuleRC',
+ 'PythonBundle',
+ 'Toolchain',
+ 'Toy_Extension',
+ 'bar',
+ ]
self.assertEqual(sorted(eb_names), expected)
def test_gen_easyblocks_overview(self):
diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
index 93d9fed793..2e4953b8bc 100644
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -284,8 +284,11 @@ def test_make_module_extend_modpath(self):
txt = eb.make_module_extend_modpath()
if module_syntax == 'Tcl':
regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses]
- home = r'\[if { \[info exists ::env\(HOME\)\] } { concat \$::env\(HOME\) } '
- home += r'else { concat "HOME_NOT_DEFINED" } \]'
+ if self.modtool.supports_tcl_getenv:
+ home = r'\[getenv HOME "HOME_NOT_DEFINED"\]'
+ else:
+ home = r'\[if { \[info exists ::env\(HOME\)\] } { concat \$::env\(HOME\) } '
+ home += r'else { concat "HOME_NOT_DEFINED" } \]'
fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir
regexs.extend([
# extension for user modules is guarded
@@ -310,7 +313,7 @@ def test_make_module_extend_modpath(self):
regex = re.compile(regex, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
- # Repeat this but using an alternate envvars (instead of $HOME)
+ # Repeat this but using an alternative envvars (instead of $HOME)
list_of_envvars = ['SITE_INSTALLS', 'USER_INSTALLS']
build_options = {
@@ -327,9 +330,12 @@ def test_make_module_extend_modpath(self):
for envvar in list_of_envvars:
if module_syntax == 'Tcl':
regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses]
- module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} ' % envvar
- module_envvar += r'\{ concat \$::env\(%s\) \} ' % envvar
- module_envvar += r'else { concat "%s" } \]' % (envvar + '_NOT_DEFINED')
+ if self.modtool.supports_tcl_getenv:
+ module_envvar = r'\[getenv %s "%s"]' % (envvar, envvar + '_NOT_DEFINED')
+ else:
+ module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} ' % envvar
+ module_envvar += r'\{ concat \$::env\(%s\) \} ' % envvar
+ module_envvar += r'else { concat "%s" } \]' % (envvar + '_NOT_DEFINED')
fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir
regexs.extend([
# extension for user modules is guarded
@@ -754,21 +760,26 @@ def test_make_module_dep(self):
eb.prepare_step()
if get_module_syntax() == 'Tcl':
- tc_load = '\n'.join([
- "if { ![ is-loaded gompi/2018a ] } {",
- " module load gompi/2018a",
- "}",
- ])
- fftw_load = '\n'.join([
- "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {",
- " module load FFTW/3.3.7-gompi-2018a",
- "}",
- ])
- lapack_load = '\n'.join([
- "if { ![ is-loaded OpenBLAS/0.2.20-GCC-6.4.0-2.28 ] } {",
- " module load OpenBLAS/0.2.20-GCC-6.4.0-2.28",
- "}",
- ])
+ if self.modtool.supports_safe_auto_load:
+ tc_load = "module load gompi/2018a"
+ fftw_load = "module load FFTW/3.3.7-gompi-2018a"
+ lapack_load = "module load OpenBLAS/0.2.20-GCC-6.4.0-2.28"
+ else:
+ tc_load = '\n'.join([
+ "if { ![ is-loaded gompi/2018a ] } {",
+ " module load gompi/2018a",
+ "}",
+ ])
+ fftw_load = '\n'.join([
+ "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {",
+ " module load FFTW/3.3.7-gompi-2018a",
+ "}",
+ ])
+ lapack_load = '\n'.join([
+ "if { ![ is-loaded OpenBLAS/0.2.20-GCC-6.4.0-2.28 ] } {",
+ " module load OpenBLAS/0.2.20-GCC-6.4.0-2.28",
+ "}",
+ ])
elif get_module_syntax() == 'Lua':
tc_load = '\n'.join([
'if not ( isloaded("gompi/2018a") ) then',
@@ -798,12 +809,18 @@ def test_make_module_dep(self):
}
if get_module_syntax() == 'Tcl':
- fftw_load = '\n'.join([
- "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {",
- " module unload FFTW",
- " module load FFTW/3.3.7-gompi-2018a",
- "}",
- ])
+ if self.modtool.supports_safe_auto_load:
+ fftw_load = '\n'.join([
+ "module unload FFTW",
+ "module load FFTW/3.3.7-gompi-2018a",
+ ])
+ else:
+ fftw_load = '\n'.join([
+ "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {",
+ " module unload FFTW",
+ " module load FFTW/3.3.7-gompi-2018a",
+ "}",
+ ])
elif get_module_syntax() == 'Lua':
fftw_load = '\n'.join([
'if not ( isloaded("FFTW/3.3.7-gompi-2018a") ) then',
@@ -856,10 +873,10 @@ def test_make_module_dep_hmns(self):
with self.mocked_stdout_stderr():
mod_dep_txt = eb.make_module_dep()
for mod in ['GCC/6.4.0-2.28', 'OpenMPI/2.1.2']:
- regex = re.compile('load.*%s' % mod)
+ regex = re.compile('(load|depends[-_]on).*%s' % mod)
self.assertFalse(regex.search(mod_dep_txt), "Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
- regex = re.compile('load.*FFTW/3.3.7')
+ regex = re.compile('(load|depends[-_]on).*FFTW/3.3.7')
self.assertTrue(regex.search(mod_dep_txt), "Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
def test_make_module_dep_of_dep_hmns(self):
@@ -1038,6 +1055,64 @@ def test_extensions_step(self):
eb.close_log()
os.remove(eb.logfile)
+ def test_extensions_step_deprecations(self):
+ """Test extension install with deprecated substeps."""
+ install_substeps = ["pre_install_extension", "install_extension", "post_install_extension"]
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ test_ec_txt = '\n'.join([
+ 'easyblock = "ConfigureMake"',
+ 'name = "pi"',
+ 'version = "3.14"',
+ 'homepage = "http://example.com"',
+ 'description = "test easyconfig"',
+ 'toolchain = SYSTEM',
+ 'exts_defaultclass = "DummyExtension"',
+ 'exts_list = ["ext1"]',
+ 'exts_list = [',
+ ' "dummy_ext",',
+ ' ("custom_ext", "0.0", {"easyblock": "CustomDummyExtension"}),',
+ ' ("deprec_ext", "0.0", {"easyblock": "DeprecatedDummyExtension"}),',
+ ' ("childcustom_ext", "0.0", {"easyblock": "ChildCustomDummyExtension"}),',
+ ' ("childdeprec_ext", "0.0", {"easyblock": "ChildDeprecatedDummyExtension"}),',
+ ']',
+ ])
+ write_file(test_ec, test_ec_txt)
+ ec = process_easyconfig(test_ec)[0]
+ eb = get_easyblock_instance(ec)
+ eb.prepare_for_extensions()
+ eb.init_ext_instances()
+
+ # Default DummyExtension without deprecated or custom install substeps
+ ext = eb.ext_instances[0]
+ self.assertEqual(ext.__class__.__name__, "DummyExtension")
+ for substep in install_substeps:
+ self.assertEqual(ext.install_extension_substep(substep), None)
+ # CustomDummyExtension
+ ext = eb.ext_instances[1]
+ self.assertEqual(ext.__class__.__name__, "CustomDummyExtension")
+ for substep in install_substeps:
+ expected_return = f"Extension installed with custom {substep}()"
+ self.assertEqual(ext.install_extension_substep(substep), expected_return)
+ # DeprecatedDummyExtension
+ ext = eb.ext_instances[2]
+ self.assertEqual(ext.__class__.__name__, "DeprecatedDummyExtension")
+ for substep in install_substeps:
+ expected_error = rf"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*"
+ self.assertErrorRegex(EasyBuildError, expected_error, ext.install_extension_substep, substep)
+ # ChildCustomDummyExtension
+ ext = eb.ext_instances[3]
+ self.assertEqual(ext.__class__.__name__, "ChildCustomDummyExtension")
+ for substep in install_substeps:
+ expected_return = f"Extension installed with custom {substep}()"
+ self.assertEqual(ext.install_extension_substep(substep), expected_return)
+ # ChildDeprecatedDummyExtension
+ ext = eb.ext_instances[4]
+ self.assertEqual(ext.__class__.__name__, "ChildDeprecatedDummyExtension")
+ for substep in install_substeps:
+ expected_error = rf"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*"
+ self.assertErrorRegex(EasyBuildError, expected_error, ext.install_extension_substep, substep)
+
def test_init_extensions(self):
"""Test creating extension instances."""
@@ -1184,6 +1259,7 @@ def test_make_module_step(self):
modextrapaths = {
'PATH': ('xbin', 'pibin'),
'CPATH': 'pi/include',
+ 'TCLLIBPATH': {'paths': 'pi', 'delimiter': ' '},
}
modextrapaths_append = {'APPEND_PATH': 'append_path'}
self.contents = '\n'.join([
@@ -1258,55 +1334,74 @@ def test_make_module_step(self):
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (key, vals) in modextrapaths.items():
- if isinstance(vals, str):
- vals = [vals]
- for val in vals:
- if get_module_syntax() == 'Tcl':
- regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
- elif get_module_syntax() == 'Lua':
- regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M)
- else:
- self.fail("Unknown module syntax: %s" % get_module_syntax())
- self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
- # Check for duplicates
- num_prepends = len(regex.findall(txt))
- self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt))
+ if isinstance(vals, dict):
+ delim = vals['delimiter']
+ paths = vals['paths']
+ if isinstance(paths, str):
+ paths = [paths]
+
+ for val in paths:
+ if get_module_syntax() == 'Tcl':
+ regex = re.compile(fr'^prepend-path\s+-d\s+"{delim}"\s+{key}\s+\$root/{val}$', re.M)
+ elif get_module_syntax() == 'Lua':
+ regex = re.compile(fr'^prepend_path\("{key}", pathJoin\(root, "{val}"\), "{delim}"\)$', re.M)
+ else:
+ self.fail("Unknown module syntax: %s" % get_module_syntax())
+ self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
+ # Check for duplicates
+ num_prepends = len(regex.findall(txt))
+ self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt))
+ else:
+ if isinstance(vals, str):
+ vals = [vals]
+
+ for val in vals:
+ if get_module_syntax() == 'Tcl':
+ regex = re.compile(fr'^prepend-path\s+{key}\s+\$root/{val}$', re.M)
+ elif get_module_syntax() == 'Lua':
+ regex = re.compile(fr'^prepend_path\("{key}", pathJoin\(root, "{val}"\)\)$', re.M)
+ else:
+ self.fail("Unknown module syntax: %s" % get_module_syntax())
+ self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
+ # Check for duplicates
+ num_prepends = len(regex.findall(txt))
+ self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt))
for (key, vals) in modextrapaths_append.items():
if isinstance(vals, str):
vals = [vals]
for val in vals:
if get_module_syntax() == 'Tcl':
- regex = re.compile(r'^append-path\s+%s\s+\$root/%s$' % (key, val), re.M)
+ regex = re.compile(r'^append-path\s+(-d ".")?%s\s+\$root/%s$' % (key, val), re.M)
elif get_module_syntax() == 'Lua':
- regex = re.compile(r'^append_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M)
+ regex = re.compile(r'^append_path\("%s", pathJoin\(root, "%s"\)(, ".")?\)$' % (key, val), re.M)
else:
self.fail("Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('GCC', '6.4.0-2.28')]:
if get_module_syntax() == 'Tcl':
- regex = re.compile(r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M)
+ regex = re.compile(r'^\s*(module load|depends-on) %s\s*$' % os.path.join(name, ver), re.M)
elif get_module_syntax() == 'Lua':
- regex = re.compile(r'^\s*load\("%s"\)$' % os.path.join(name, ver), re.M)
+ regex = re.compile(r'^\s*(load|depends_on)\("%s"\)$' % os.path.join(name, ver), re.M)
else:
self.fail("Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('test', '1.2.3')]:
if get_module_syntax() == 'Tcl':
- regex = re.compile(r'^\s*module load %s/.%s\s*$' % (name, ver), re.M)
+ regex = re.compile(r'^\s*(module load|depends-on) %s/.%s\s*$' % (name, ver), re.M)
elif get_module_syntax() == 'Lua':
- regex = re.compile(r'^\s*load\("%s/.%s"\)$' % (name, ver), re.M)
+ regex = re.compile(r'^\s*(load|depends_on)\("%s/.%s"\)$' % (name, ver), re.M)
else:
self.fail("Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('OpenMPI', '2.1.2-GCC-6.4.0-2.28')]:
if get_module_syntax() == 'Tcl':
- regex = re.compile(r'^\s*module load %s/.?%s\s*$' % (name, ver), re.M)
+ regex = re.compile(r'^\s*(module load|depends-on) %s/.?%s\s*$' % (name, ver), re.M)
elif get_module_syntax() == 'Lua':
- regex = re.compile(r'^\s*load\("%s/.?%s"\)$' % (name, ver), re.M)
+ regex = re.compile(r'^\s*(load|depends_on)\("%s/.?%s"\)$' % (name, ver), re.M)
else:
self.fail("Unknown module syntax: %s" % get_module_syntax())
self.assertFalse(regex.search(txt), "Pattern '%s' *not* found in %s" % (regex.pattern, txt))
@@ -1478,9 +1573,8 @@ def test_fetch_sources(self):
self.assertTrue(os.path.samefile(eb.src[0]['path'], toy_source))
self.assertEqual(eb.src[0]['name'], 'toy-0.0.tar.gz')
self.assertEqual(eb.src[0]['cmd'], None)
- self.assertEqual(len(eb.src[0]['checksum']), 7)
- self.assertEqual(eb.src[0]['checksum'][0], 'be662daa971a640e40be5c804d9d7d10')
- self.assertEqual(eb.src[0]['checksum'][1], '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc')
+ self.assertEqual(len(eb.src[0]['checksum']), 2)
+ self.assertEqual(eb.src[0]['checksum'][0], '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc')
# reconfigure EasyBuild so we can check 'downloaded' sources
os.environ['EASYBUILD_SOURCEPATH'] = self.test_prefix
@@ -1688,7 +1782,7 @@ def test_fetch_patches(self):
self.assertEqual(eb.patches[1]['level'], 4)
self.assertEqual(eb.patches[2]['name'], toy_patch)
self.assertEqual(eb.patches[2]['sourcepath'], 'foobar')
- self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz'),
+ self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz')
self.assertEqual(eb.patches[3]['copy'], 'some/path')
self.assertEqual(eb.patches[4]['name'], toy_patch)
self.assertEqual(eb.patches[4]['level'], 0)
@@ -1729,7 +1823,7 @@ def test_obtain_file(self):
# test no_download option
urls = ['file://%s' % tmpdir_subdir]
- error_pattern = "Couldn't find file toy-0.0.tar.gz anywhere, and downloading it is disabled"
+ error_pattern = "Couldn't find file 'toy-0.0.tar.gz' anywhere, and downloading it is disabled"
with self.mocked_stdout_stderr():
self.assertErrorRegex(EasyBuildError, error_pattern, eb.obtain_file,
toy_tarball, urls=urls, alt_location='alt_toy', no_download=True)
@@ -1756,7 +1850,7 @@ def test_obtain_file(self):
res = eb.obtain_file(toy_tarball)
self.assertEqual(res, toy_tarball_path)
- # finding a file in the alternate location works
+ # finding a file in the alternative location works
with self.mocked_stdout_stderr():
res = eb.obtain_file(toy_tarball, alt_location='alt_toy')
self.assertEqual(res, alt_toy_tarball_path)
@@ -2127,7 +2221,7 @@ def test_parallel(self):
handle, toy_ec1 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
os.close(handle)
- write_file(toy_ec1, toytxt + "\nparallel = 123")
+ write_file(toy_ec1, toytxt + "\nparallel = 13")
handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
os.close(handle)
@@ -2145,7 +2239,7 @@ def test_parallel(self):
# only 'parallel' easyconfig parameter specified (no 'parallel' build option)
test_eb = EasyBlock(EasyConfig(toy_ec1))
test_eb.check_readiness_step()
- self.assertEqual(test_eb.cfg['parallel'], 123)
+ self.assertEqual(test_eb.cfg['parallel'], 13)
# both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option)
test_eb = EasyBlock(EasyConfig(toy_ec2))
@@ -2158,20 +2252,20 @@ def test_parallel(self):
self.assertEqual(test_eb.cfg['parallel'], False)
# only 'parallel' build option specified
- init_config(build_options={'parallel': '97', 'validate': False})
+ init_config(build_options={'parallel': '9', 'validate': False})
test_eb = EasyBlock(EasyConfig(toy_ec))
test_eb.check_readiness_step()
- self.assertEqual(test_eb.cfg['parallel'], 97)
+ self.assertEqual(test_eb.cfg['parallel'], 9)
# both 'parallel' build option and easyconfig parameter specified (no 'maxparallel')
test_eb = EasyBlock(EasyConfig(toy_ec1))
test_eb.check_readiness_step()
- self.assertEqual(test_eb.cfg['parallel'], 97)
+ self.assertEqual(test_eb.cfg['parallel'], 9)
# both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option
test_eb = EasyBlock(EasyConfig(toy_ec2))
test_eb.check_readiness_step()
- self.assertEqual(test_eb.cfg['parallel'], 67)
+ self.assertEqual(test_eb.cfg['parallel'], 9)
# make sure 'parallel = False' is not overriden (with 'parallel' build option)
test_eb = EasyBlock(EasyConfig(toy_ec3))
@@ -2222,18 +2316,25 @@ def test_extension_set_start_dir(self):
cwd = os.getcwd()
self.assertExists(cwd)
- def check_ext_start_dir(expected_start_dir, unpack_src=True):
+ def check_ext_start_dir(expected_start_dir, unpack_src=True, parent_startdir=None):
"""Check start dir."""
# make sure we're in an existing directory at the start
change_dir(cwd)
+
eb = EasyBlock(ec['ec'])
+ if not os.path.exists(eb.builddir):
+ eb.make_builddir() # Required to exist for samefile
+ eb.cfg['start_dir'] = parent_startdir
+
eb.extensions_step(fetch=True, install=False)
# extract sources of the extension
ext = eb.ext_instances[-1]
- ext.run(unpack_src=unpack_src)
+ ext.install_extension(unpack_src=unpack_src)
if expected_start_dir is None:
self.assertIsNone(ext.start_dir)
+ # Without a start dir we don't change the CWD
+ self.assertEqual(os.getcwd(), cwd)
else:
self.assertTrue(os.path.isabs(ext.start_dir))
if ext.start_dir != os.sep:
@@ -2243,14 +2344,8 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True):
else:
abs_expected_start_dir = os.path.join(eb.builddir, expected_start_dir)
self.assertEqual(ext.start_dir, abs_expected_start_dir)
- if not os.path.exists(eb.builddir):
- eb.make_builddir() # Required to exist for samefile
self.assertTrue(os.path.samefile(ext.start_dir, abs_expected_start_dir))
- if unpack_src:
self.assertTrue(os.path.samefile(os.getcwd(), abs_expected_start_dir))
- else:
- # When not unpacking we don't change the CWD
- self.assertEqual(os.getcwd(), cwd)
remove_dir(eb.builddir)
ec['ec']['exts_defaultclass'] = 'DummyExtension'
@@ -2279,11 +2374,8 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True):
'start_dir': 'nonexistingdir'}),
]
with self.mocked_stdout_stderr():
- err_pattern = "Failed to change from .*barbar/barbar-0.0 to nonexistingdir.*"
+ err_pattern = r"Provided start dir \(nonexistingdir\) for extension barbar does not exist:.*"
self.assertErrorRegex(EasyBuildError, err_pattern, check_ext_start_dir, 'whatever')
- stderr = self.get_stderr()
- warning_pattern = "WARNING: Provided start dir (nonexistingdir) for extension barbar does not exist"
- self.assertIn(warning_pattern, stderr)
# No error when using relative path in non-extracted source for some reason
ec['ec']['exts_list'] = [
@@ -2313,6 +2405,15 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True):
check_ext_start_dir(os.sep, unpack_src=False)
self.assertFalse(self.get_stderr())
+ # Go to ECs start dir if nosource is used
+ ec['ec']['exts_list'] = [
+ ('barbar', '0.0', {
+ 'nosource': True}),
+ ]
+ with self.mocked_stdout_stderr():
+ check_ext_start_dir(self.test_prefix, parent_startdir=self.test_prefix)
+ self.assertFalse(self.get_stderr())
+
def test_prepare_step(self):
"""Test prepare step (setting up build environment)."""
test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
@@ -2508,8 +2609,8 @@ def test_checksum_step(self):
copy_file(toy_ec, self.test_prefix)
toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec))
ectxt = read_file(toy_ec)
- # replace MD5 checksum for toy-0.0.tar.gz
- ectxt = ectxt.replace('be662daa971a640e40be5c804d9d7d10', '00112233445566778899aabbccddeeff')
+ # replace SHA256 checksum for toy-0.0.tar.gz
+ ectxt = ectxt.replace('44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', '76543210' * 8)
# replace SHA256 checksums for source of bar extension
ectxt = ectxt.replace('f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7', '01234567' * 8)
write_file(toy_ec, ectxt)
@@ -2599,7 +2700,8 @@ def test_checksum_step(self):
self.mock_stderr(False)
self.mock_stdout(False)
self.assertEqual(stdout, '')
- self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz")
+ self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz\n\n\n"
+ "WARNING: Ignoring failing checksum verification for toy-0.0.tar.gz")
def test_check_checksums(self):
"""Test for check_checksums_for and check_checksums methods."""
@@ -2655,10 +2757,10 @@ def run_checks():
# no checksum issues
self.assertEqual(eb.check_checksums(), [])
- # tuple of two alternate SHA256 checksums: OK
+ # tuple of two alternative SHA256 checksums: OK
eb.cfg['checksums'] = [
(
- # two alternate checksums for toy-0.0.tar.gz
+ # two alternative checksums for toy-0.0.tar.gz
'a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd',
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',
),
diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py
index 141e9b0f77..1e630a5627 100644
--- a/test/framework/easyconfig.py
+++ b/test/framework/easyconfig.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,7 +40,6 @@
import textwrap
from collections import OrderedDict
from easybuild.tools import LooseVersion
-from importlib import reload
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from unittest import TextTestRunner
@@ -86,11 +85,7 @@
try:
import pycodestyle # noqa
except ImportError:
- try:
- import pep8 # noqa
- except ImportError:
- pass
-
+ pass
EXPECTED_DOTTXT_TOY_DEPS = """digraph graphname {
toy;
@@ -120,6 +115,11 @@ def setUp(self):
github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT)
self.skip_github_tests = github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None
+ self.orig_easyconfig_DEPRECATED_EASYCONFIG_PARAMETERS = easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS
+ self.orig_easyconfig_DEPRECATED_EASYCONFIG_TEMPLATES = easyconfig.easyconfig.DEPRECATED_EASYCONFIG_TEMPLATES
+ self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_PARAMETERS = easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS
+ self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_TEMPLATES = easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_TEMPLATES
+
def prep(self):
"""Prepare for test."""
# (re)cleanup last test file
@@ -133,16 +133,26 @@ def prep(self):
def tearDown(self):
""" make sure to remove the temporary file """
st.get_cpu_architecture = self.orig_get_cpu_architecture
+
super(EasyConfigTest, self).tearDown()
if os.path.exists(self.eb_file):
os.remove(self.eb_file)
+ # restore orignal values of DEPRECATED_EASYCONFIG_TEMPLATES & co in easyconfig.templates
+ easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS = self.orig_easyconfig_DEPRECATED_EASYCONFIG_PARAMETERS
+ easyconfig.easyconfig.DEPRECATED_EASYCONFIG_TEMPLATES = self.orig_easyconfig_DEPRECATED_EASYCONFIG_TEMPLATES
+ easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS = self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_PARAMETERS
+ easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_TEMPLATES = self.orig_easyconfig_ALTERNATIVE_EASYCONFIG_TEMPLATES
+
def test_empty(self):
""" empty files should not parse! """
+ self.assertErrorRegex(EasyBuildError, "expected a valid path", EasyConfig, "")
self.contents = "# empty string"
self.prep()
self.assertRaises(EasyBuildError, EasyConfig, self.eb_file)
- self.assertErrorRegex(EasyBuildError, "expected a valid path", EasyConfig, "")
+ self.contents = ""
+ self.prep()
+ self.assertErrorRegex(EasyBuildError, "is empty", EasyConfig, self.eb_file)
def test_mandatory(self):
""" make sure all checking of mandatory parameters works """
@@ -485,7 +495,7 @@ def test_exts_list(self):
# SHA256 checksum for source (gzip-1.4.eb)
"6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45",
# SHA256 checksum for 'patch' (toy-0.0.eb)
- "2d964e0e8f05a7cce0dd83a3e68c9737da14b87b61b8b8b0291d58d4c8d1031c",
+ "177b34bcdfa1abde96f30354848a01894ebc9c24913bc5145306cd30f78fc8ad",
],
}),
# Can use templates in name and version
@@ -509,7 +519,7 @@ def test_exts_list(self):
self.assertEqual(exts_sources[1]['version'], '2.0')
self.assertEqual(exts_sources[1]['options'], {
'checksums': ['6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45',
- '2d964e0e8f05a7cce0dd83a3e68c9737da14b87b61b8b8b0291d58d4c8d1031c'],
+ '177b34bcdfa1abde96f30354848a01894ebc9c24913bc5145306cd30f78fc8ad'],
'patches': [('toy-0.0.eb', '.')],
'source_tmpl': 'gzip-1.4.eb',
'source_urls': [('http://example.com', 'suffix')],
@@ -522,7 +532,8 @@ def test_exts_list(self):
with self.mocked_stdout_stderr():
modfile = os.path.join(eb.make_module_step(), 'PI', '3.14' + eb.module_generator.MODULE_FILE_EXTENSION)
modtxt = read_file(modfile)
- regex = re.compile('EBEXTSLISTPI.*ext1-1.0,ext2-2.0')
+ # verify that templates used for extensions are resolved as they should
+ regex = re.compile('EBEXTSLISTPI.*"ext1-1.0,ext2-2.0,ext-PI-3.14,ext-pi-3.0')
self.assertTrue(regex.search(modtxt), "Pattern '%s' found in: %s" % (regex.pattern, modtxt))
def test_extensions_templates(self):
@@ -737,6 +748,40 @@ def test_tweaking(self):
# cleanup
os.remove(tweaked_fn)
+ def test_alt_easyconfig_paths(self):
+ """Test alt_easyconfig_paths function that collects list of additional paths for easyconfig files."""
+
+ tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix)
+ self.assertEqual(tweaked_ecs_path, None)
+ self.assertEqual(extra_ecs_path, [])
+
+ tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True)
+ self.assertTrue(tweaked_ecs_path)
+ self.assertTrue(isinstance(tweaked_ecs_path, tuple))
+ self.assertEqual(len(tweaked_ecs_path), 2)
+ self.assertEqual(tweaked_ecs_path[0], os.path.join(self.test_prefix, 'tweaked_easyconfigs'))
+ self.assertEqual(tweaked_ecs_path[1], os.path.join(self.test_prefix, 'tweaked_dep_easyconfigs'))
+ self.assertEqual(extra_ecs_path, [])
+
+ tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123, 456])
+ self.assertEqual(tweaked_ecs_path, None)
+ self.assertTrue(extra_ecs_path)
+ self.assertTrue(isinstance(extra_ecs_path, list))
+ self.assertEqual(len(extra_ecs_path), 2)
+ self.assertEqual(extra_ecs_path[0], os.path.join(self.test_prefix, 'files_pr123'))
+ self.assertEqual(extra_ecs_path[1], os.path.join(self.test_prefix, 'files_pr456'))
+
+ tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123, 456],
+ review_pr=789, from_commit='c0ff33')
+ self.assertEqual(tweaked_ecs_path, None)
+ self.assertTrue(extra_ecs_path)
+ self.assertTrue(isinstance(extra_ecs_path, list))
+ self.assertEqual(len(extra_ecs_path), 4)
+ self.assertEqual(extra_ecs_path[0], os.path.join(self.test_prefix, 'files_pr123'))
+ self.assertEqual(extra_ecs_path[1], os.path.join(self.test_prefix, 'files_pr456'))
+ self.assertEqual(extra_ecs_path[2], os.path.join(self.test_prefix, 'files_pr789'))
+ self.assertEqual(extra_ecs_path[3], os.path.join(self.test_prefix, 'files_commit_c0ff33'))
+
def test_tweak_multiple_tcs(self):
"""Test that tweaking variables of ECs from multiple toolchains works"""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
@@ -1226,6 +1271,14 @@ def test_templating_constants(self):
ec = EasyConfig(test_ec)
self.assertEqual(ec['sanity_check_commands'], ['mpiexec -np 1 -- toy'])
+ def test_template_constant_import(self):
+ """Test importing template constants works"""
+ from easybuild.framework.easyconfig.templates import GITHUB_SOURCE, GNU_SOURCE, SHLIB_EXT
+ from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS
+ self.assertEqual(GITHUB_SOURCE, TEMPLATE_CONSTANTS['GITHUB_SOURCE'][0])
+ self.assertEqual(GNU_SOURCE, TEMPLATE_CONSTANTS['GNU_SOURCE'][0])
+ self.assertEqual(SHLIB_EXT, get_shared_lib_ext())
+
def test_templating_cuda_toolchain(self):
"""Test templates via toolchain component, like setting %(cudaver)s with fosscuda toolchain."""
@@ -1333,7 +1386,7 @@ def test_templating_doc(self):
# expected length: 1 per constant and 2 extra per constantgroup (title + empty line in between)
temps = [
easyconfig.templates.TEMPLATE_NAMES_EASYCONFIG,
- easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 3,
+ list(easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS.keys()) * 3,
easyconfig.templates.TEMPLATE_NAMES_CONFIG,
easyconfig.templates.TEMPLATE_NAMES_LOWER,
easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP,
@@ -1389,6 +1442,30 @@ def test_start_dir_template(self):
self.assertIn('start_dir in extension configure is %s &&' % ext_start_dir, logtxt)
self.assertIn('start_dir in extension build is %s &&' % ext_start_dir, logtxt)
+ def test_rpath_template(self):
+ """Test the %(rpath)s template"""
+ test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt += "configopts = '--with-rpath=%(rpath_enabled)s'"
+ write_file(test_ec, test_ec_txt)
+
+ ec = EasyConfig(test_ec)
+ expected = '--with-rpath=true' if get_os_name() == 'Linux' else '--with-rpath=false'
+ self.assertEqual(ec['configopts'], expected)
+
+ # force True
+ update_build_option('rpath', True)
+ ec = EasyConfig(test_ec)
+ self.assertEqual(ec['configopts'], "--with-rpath=true")
+
+ # force False
+ update_build_option('rpath', False)
+ ec = EasyConfig(test_ec)
+ self.assertEqual(ec['configopts'], "--with-rpath=false")
+
def test_sysroot_template(self):
"""Test the %(sysroot)s template"""
@@ -1417,6 +1494,76 @@ def test_sysroot_template(self):
self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix)
self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix)
+ def test_software_commit_template(self):
+ """Test the %(software_commit)s template"""
+
+ test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt += '\nconfigopts = "--some-opt=%(software_commit)s"'
+ test_ec_txt += '\nbuildopts = "--some-opt=%(software_commit)s"'
+ test_ec_txt += '\ninstallopts = "--some-opt=%(software_commit)s"'
+ write_file(test_ec, test_ec_txt)
+
+ # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None)
+ ec = EasyConfig(test_ec)
+ self.assertEqual(ec['configopts'], "--some-opt=")
+ self.assertEqual(ec['buildopts'], "--some-opt=")
+ self.assertEqual(ec['installopts'], "--some-opt=")
+
+ # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None)
+ # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist
+ software_commit = '1234bc'
+ update_build_option('software_commit', software_commit)
+
+ ec = EasyConfig(test_ec)
+ self.assertEqual(ec['configopts'], "--some-opt=%s" % software_commit)
+ self.assertEqual(ec['buildopts'], "--some-opt=%s" % software_commit)
+ self.assertEqual(ec['installopts'], "--some-opt=%s" % software_commit)
+
+ def test_template_deprecation_and_alternative(self):
+ """Test deprecation of (and alternative) templates"""
+
+ self.prep()
+
+ template_test_deprecations = {
+ 'builddir': ('depr_build_dir', '1000000000'),
+ 'cudaver': ('depr_cuda_ver', '1000000000'),
+ 'start_dir': ('depr_start_dir', '1000000000'),
+ }
+ easyconfig.easyconfig.DEPRECATED_EASYCONFIG_TEMPLATES = template_test_deprecations
+
+ template_test_alternatives = {
+ 'installdir': 'alt_install_dir',
+ 'version_major_minor': 'alt_ver_maj_min',
+ }
+ easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_TEMPLATES = template_test_alternatives
+
+ tmpl_str = ("cd %(start_dir)s && make %(namelower)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'"
+ " && echo %(alt_install_dir)s %(version_major_minor)s")
+ tmpl_dict = {
+ 'depr_build_dir': '/example/build_dir',
+ 'depr_cuda_ver': '12.1.1',
+ 'installdir': '/example/installdir',
+ 'start_dir': '/example/build_dir/start_dir',
+ 'alt_ver_maj_min': '1.2',
+ 'namelower': 'foo',
+ }
+
+ with self.mocked_stdout_stderr() as (_, stderr):
+ res = resolve_template(tmpl_str, tmpl_dict)
+ stderr = stderr.getvalue()
+
+ for tmpl in [*template_test_deprecations.keys(), *template_test_alternatives.keys()]:
+ self.assertNotIn("%(" + tmpl + ")s", res)
+
+ for old, (new, ver) in template_test_deprecations.items():
+ depr_str = (f"WARNING: Deprecated functionality, will no longer work in EasyBuild v{ver}: "
+ f"Easyconfig template '{old}' is deprecated, use '{new}' instead")
+ self.assertIn(depr_str, stderr)
+
def test_constant_doc(self):
"""test constant documentation"""
doc = avail_easyconfig_constants()
@@ -1605,6 +1752,10 @@ def test_get_easyblock_class(self):
self.assertErrorRegex(EasyBuildError, "Failed to import EB_TOY", get_easyblock_class, None, name='TOY')
self.assertEqual(get_easyblock_class(None, name='TOY', error_on_failed_import=False), None)
+ # Test passing neither easyblock nor name
+ self.assertErrorRegex(EasyBuildError, "neither name nor easyblock were specified", get_easyblock_class, None)
+ self.assertEqual(get_easyblock_class(None, error_on_missing_easyblock=False), None)
+
def test_letter_dir(self):
"""Test letter_dir_for function."""
test_cases = {
@@ -1732,8 +1883,54 @@ def foo(key):
self.assertErrorRegex(EasyBuildError, error_regex, foo, key)
+ def test_alternative_easyconfig_parameters(self):
+ """Test handling of alternative easyconfig parameters."""
+
+ test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')
+
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt = test_ec_txt.replace('postinstallcmds', 'post_install_cmds')
+ test_ec_txt = test_ec_txt.replace('moduleclass', 'env_mod_class')
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, test_ec_txt)
+
+ # post_install_cmds is not accepted unless it's registered as an alternative easyconfig parameter
+ easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS = {}
+ self.assertErrorRegex(EasyBuildError, "post_install_cmds -> postinstallcmds", EasyConfig, test_ec)
+
+ easyconfig.easyconfig.ALTERNATIVE_EASYCONFIG_PARAMETERS = {
+ 'env_mod_class': 'moduleclass',
+ 'post_install_cmds': 'postinstallcmds',
+ }
+ ec = EasyConfig(test_ec)
+
+ expected = 'tools'
+ self.assertEqual(ec['moduleclass'], expected)
+ self.assertEqual(ec['env_mod_class'], expected)
+
+ expected = ['echo TOY > %(installdir)s/README']
+ self.assertEqual(ec['postinstallcmds'], expected)
+ self.assertEqual(ec['post_install_cmds'], expected)
+
+ # test setting of easyconfig parameter with original & alternative name
+ ec['moduleclass'] = 'test1'
+ self.assertEqual(ec['moduleclass'], 'test1')
+ self.assertEqual(ec['env_mod_class'], 'test1')
+ ec.update('moduleclass', 'test2')
+ self.assertEqual(ec['moduleclass'], 'test1 test2 ')
+ self.assertEqual(ec['env_mod_class'], 'test1 test2 ')
+
+ ec['env_mod_class'] = 'test3'
+ self.assertEqual(ec['moduleclass'], 'test3')
+ self.assertEqual(ec['env_mod_class'], 'test3')
+ ec.update('env_mod_class', 'test4')
+ self.assertEqual(ec['moduleclass'], 'test3 test4 ')
+ self.assertEqual(ec['env_mod_class'], 'test3 test4 ')
+
def test_deprecated_easyconfig_parameters(self):
- """Test handling of replaced easyconfig parameters."""
+ """Test handling of deprecated easyconfig parameters."""
os.environ.pop('EASYBUILD_DEPRECATED')
easybuild.tools.build_log.CURRENT_VERSION = self.orig_current_version
init_config()
@@ -1741,18 +1938,15 @@ def test_deprecated_easyconfig_parameters(self):
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb'))
- orig_deprecated_parameters = copy.deepcopy(easyconfig.parser.DEPRECATED_PARAMETERS)
- easyconfig.parser.DEPRECATED_PARAMETERS.update({
+ easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS = {
'foobar': ('barfoo', '0.0'), # deprecated since forever
- 'foobarbarfoo': ('barfoofoobar', '1000000000'), # won't be actually deprecated for a while
- })
-
- # copy classes before reloading, so we can restore them (other isinstance checks fail)
- orig_EasyConfig = copy.deepcopy(easyconfig.easyconfig.EasyConfig)
- orig_ActiveMNS = copy.deepcopy(easyconfig.easyconfig.ActiveMNS)
- reload(easyconfig.parser)
+ # won't be actually deprecated for a while;
+ # note that we should map foobarbarfoo to a valid easyconfig parameter here,
+ # or we'll hit errors when parsing an easyconfig file that uses it
+ 'foobarbarfoo': ('required_linked_shared_libs', '1000000000'),
+ }
- for key, (newkey, depr_ver) in easyconfig.parser.DEPRECATED_PARAMETERS.items():
+ for key, (newkey, depr_ver) in easyconfig.easyconfig.DEPRECATED_EASYCONFIG_PARAMETERS.items():
if LooseVersion(depr_ver) <= easybuild.tools.build_log.CURRENT_VERSION:
# deprecation error
error_regex = "DEPRECATED.*since v%s.*'%s' is deprecated.*use '%s' instead" % (depr_ver, key, newkey)
@@ -1765,18 +1959,42 @@ def foo(key):
self.assertErrorRegex(EasyBuildError, error_regex, foo, key)
else:
# only deprecation warning, but key is replaced when getting/setting
- ec[key] = 'test123'
- self.assertEqual(ec[newkey], 'test123')
- self.assertEqual(ec[key], 'test123')
- ec[newkey] = '123test'
- self.assertEqual(ec[newkey], '123test')
- self.assertEqual(ec[key], '123test')
-
- easyconfig.parser.DEPRECATED_PARAMETERS = orig_deprecated_parameters
- reload(easyconfig.parser)
- reload(easyconfig.easyconfig)
- easyconfig.easyconfig.EasyConfig = orig_EasyConfig
- easyconfig.easyconfig.ActiveMNS = orig_ActiveMNS
+ with self.mocked_stdout_stderr():
+ ec[key] = 'test123'
+ self.assertEqual(ec[newkey], 'test123')
+ self.assertEqual(ec[key], 'test123')
+ ec[newkey] = '123test'
+ self.assertEqual(ec[newkey], '123test')
+ self.assertEqual(ec[key], '123test')
+
+ variables = {
+ 'name': 'example',
+ 'version': '1.2.3',
+ 'foobar': 'foobar',
+ 'local_var': 'test',
+ }
+ ec = {
+ 'name': None,
+ 'version': None,
+ 'homepage': None,
+ 'toolchain': None,
+ }
+ ec_params, unknown_keys = triage_easyconfig_params(variables, ec)
+ # deprecated easyconfig parameter 'foobar' is retained as easyconfig parameter;
+ # only local_var is not retained, since that's a local variable
+ self.assertEqual(unknown_keys, [])
+ expected = {'name': 'example', 'version': '1.2.3', 'foobar': 'foobar'}
+ self.assertEqual(ec_params, expected)
+
+ # try parsing an easyconfig file that defines a deprecated easyconfig parameter
+ toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, read_file(toy_ec))
+ write_file(test_ec, "\nfoobarbarfoo = 'foobarbarfoo'", append=True)
+
+ with self.mocked_stdout_stderr():
+ ec = EasyConfig(test_ec)
+ self.assertEqual(ec['required_linked_shared_libs'], 'foobarbarfoo')
def test_unknown_easyconfig_parameter(self):
"""Check behaviour when unknown easyconfig parameters are used."""
@@ -2102,8 +2320,8 @@ def test_external_dependencies_templates(self):
'pyshortver': '3.6',
'pyver': '3.6.5',
}
- for key in expected_template_values:
- self.assertEqual(ec.template_values[key], expected_template_values[key])
+ for key, expected in expected_template_values.items():
+ self.assertEqual(ec.template_values[key], expected)
self.assertEqual(ec['versionsuffix'], '-Python-3.6.5-Perl-5.30')
@@ -2179,8 +2397,8 @@ def test_quote_str(self):
'foo\\bar': '"foo\\bar"',
}
- for t in teststrings:
- self.assertEqual(quote_str(t), teststrings[t])
+ for t, expected in teststrings.items():
+ self.assertEqual(quote_str(t), expected)
# test escape_newline
self.assertEqual(quote_str("foo\nbar", escape_newline=False), '"foo\nbar"')
@@ -2462,8 +2680,8 @@ def test_dump_autopep8(self):
def test_dump_extra(self):
"""Test EasyConfig's dump() method for files containing extra values"""
- if not ('pycodestyle' in sys.modules or 'pep8' in sys.modules):
- print("Skipping test_dump_extra (no pycodestyle or pep8 available)")
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_dump_extra pycodestyle is not available")
return
rawtxt = '\n'.join([
@@ -2505,8 +2723,8 @@ def test_dump_extra(self):
def test_dump_template(self):
""" Test EasyConfig's dump() method for files containing templates"""
- if not ('pycodestyle' in sys.modules or 'pep8' in sys.modules):
- print("Skipping test_dump_template (no pycodestyle or pep8 available)")
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_dump_template pycodestyle is not available")
return
rawtxt = '\n'.join([
@@ -2594,8 +2812,8 @@ def test_dump_template(self):
def test_dump_comments(self):
""" Test dump() method for files containing comments """
- if not ('pycodestyle' in sys.modules or 'pep8' in sys.modules):
- print("Skipping test_dump_comments (no pycodestyle or pep8 available)")
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_dump_comments pycodestyle is not available")
return
rawtxt = '\n'.join([
@@ -3289,6 +3507,8 @@ def test_template_constant_dict(self):
arch_regex = re.compile('^[a-z0-9_]+$')
+ rpath = 'true' if get_os_name() == 'Linux' else 'false'
+
expected = {
'bitbucket_account': 'gzip',
'github_account': 'gzip',
@@ -3298,6 +3518,8 @@ def test_template_constant_dict(self):
'nameletter': 'g',
'nameletterlower': 'g',
'parallel': None,
+ 'rpath_enabled': rpath,
+ 'software_commit': '',
'sysroot': '',
'toolchain_name': 'foss',
'toolchain_version': '2018a',
@@ -3323,7 +3545,7 @@ def test_template_constant_dict(self):
except AttributeError:
pass # Ignore if not present
orig_get_avail_core_count = st.get_avail_core_count
- st.get_avail_core_count = lambda: 42
+ st.get_avail_core_count = lambda: 12
# also check template values after running check_readiness_step (which runs set_parallel)
eb = EasyBlock(ec)
@@ -3334,7 +3556,7 @@ def test_template_constant_dict(self):
res = template_constant_dict(ec)
res.pop('arch')
- expected['parallel'] = 42
+ expected['parallel'] = 12
self.assertEqual(res, expected)
toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-deps.eb')
@@ -3381,6 +3603,8 @@ def test_template_constant_dict(self):
'pyminver': '7',
'pyshortver': '3.7',
'pyver': '3.7.2',
+ 'rpath_enabled': rpath,
+ 'software_commit': '',
'sysroot': '',
'version': '0.01',
'version_major': '0',
@@ -3446,6 +3670,8 @@ def test_template_constant_dict(self):
'namelower': 'foo',
'nameletter': 'f',
'nameletterlower': 'f',
+ 'rpath_enabled': rpath,
+ 'software_commit': '',
'sysroot': '',
'version': '1.2.3',
'version_major': '1',
@@ -3521,6 +3747,7 @@ def test_hidden_toolchain(self):
args = [
ec_file,
'--dry-run',
+ '--robot',
]
outtxt = self.eb_main(args, raise_error=True)
self.assertTrue(re.search(r'module: GCC/\.4\.9\.2', outtxt))
@@ -3616,10 +3843,34 @@ def test_resolve_template(self):
# '%(name)' is not a correct template spec (missing trailing 's')
self.assertEqual(resolve_template('%(name)', tmpl_dict), '%(name)')
+ # Correct (un)escaping
+ values = (
+ ('10%', '10%'),
+ ('%of', '%of'),
+ ('10%of', '10%of'),
+ ('%s', '%s'),
+ ('%%(name)s', '%(name)s'),
+ ('%%%(name)s', '%FooBar'),
+ ('%%%%(name)s', '%%(name)s'),
+ # It doesn't matter what is resolved
+ ('%%(invalid)s', '%(invalid)s'),
+ ('%%%%(invalid)s', '%%(invalid)s'),
+ )
+ for value, expected in values:
+ self.assertEqual(resolve_template(value, tmpl_dict), expected)
+ # Templates are resolved
+ value += ' %(name)s'
+ expected += ' FooBar'
+ self.assertEqual(resolve_template(value, tmpl_dict), expected)
+
+ # On unknown values the value is returned unchanged
+ for value in ('%(invalid)s', '%(name)s %(invalid)s', '%%%(invalid)s', '% %(invalid)s', '%s %(invalid)s'):
+ self.assertEqual(resolve_template(value, tmpl_dict), value)
+
def test_det_subtoolchain_version(self):
"""Test det_subtoolchain_version function"""
_, all_tc_classes = search_toolchain('')
- subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes)
+ subtoolchains = {tc_class.NAME: getattr(tc_class, 'SUBTOOLCHAIN', None) for tc_class in all_tc_classes}
optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False))
current_tc = {'name': 'fosscuda', 'version': '2018a'}
@@ -4288,15 +4539,19 @@ def test_triage_easyconfig_params(self):
self.assertEqual(sorted(unknown_keys), ['bleh', 'foobar'])
# check behaviour when easyconfig parameters that use a name indicating a local variable were defined
- ec.update({
+ local_vars = {
'x': None,
'local_foo': None,
'_foo': None,
'_': None,
- })
+ }
+ ec.update(local_vars)
error = "Found 4 easyconfig parameters that are considered local variables: _, _foo, local_foo, x"
self.assertErrorRegex(EasyBuildError, error, triage_easyconfig_params, variables, ec)
+ for key in local_vars:
+ del ec[key]
+
def test_local_vars_detection(self):
"""Test detection of using unknown easyconfig parameters that are likely local variables."""
@@ -4443,7 +4698,7 @@ def test_cuda_compute_capabilities(self):
prebuildopts = '%(cuda_cc_semicolon_sep)s'
buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" '
'semi="%(cuda_int_semicolon_sep)s"')
- preinstallopts = '%(cuda_cc_space_sep)s'
+ preinstallopts = 'period="%(cuda_cc_space_sep)s" noperiod="%(cuda_cc_space_sep_no_period)s"'
installopts = '%(cuda_compute_capabilities)s'
""")
self.prep()
@@ -4456,7 +4711,7 @@ def test_cuda_compute_capabilities(self):
self.assertEqual(ec['buildopts'], 'comma="51,70,71" '
'space="51 70 71" '
'semi="51;70;71"')
- self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1')
+ self.assertEqual(ec['preinstallopts'], 'period="5.1 7.0 7.1" noperiod="51 70 71"')
self.assertEqual(ec['installopts'], '5.1,7.0,7.1')
# build options overwrite it
@@ -4469,7 +4724,7 @@ def test_cuda_compute_capabilities(self):
'space="42 63" '
'semi="42;63"')
self.assertEqual(ec['prebuildopts'], '4.2;6.3')
- self.assertEqual(ec['preinstallopts'], '4.2 6.3')
+ self.assertEqual(ec['preinstallopts'], 'period="4.2 6.3" noperiod="42 63"')
self.assertEqual(ec['installopts'], '4.2,6.3')
def test_det_copy_ec_specs(self):
@@ -4560,6 +4815,10 @@ def test_recursive_module_unload(self):
toy_ec = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb')
test_ec = os.path.join(self.test_prefix, 'test.eb')
test_ec_txt = read_file(toy_ec)
+
+ # this test only makes sense if depends_on is not used
+ self.allow_deprecated_behaviour()
+ test_ec_txt += '\nmodule_depends_on = False'
write_file(test_ec, test_ec_txt)
test_module = os.path.join(self.test_installpath, 'modules', 'all', 'foss', '2018a')
@@ -4570,7 +4829,10 @@ def test_recursive_module_unload(self):
recursive_unload_pat = r'if mode\(\) == "unload" or not \( isloaded\("%(mod)s"\) \) then\n'
recursive_unload_pat += r'\s*load\("%(mod)s"\)'
else:
- guarded_load_pat = r'if { \!\[ is-loaded %(mod)s \] } {\n\s*module load %(mod)s'
+ if self.modtool.supports_safe_auto_load:
+ guarded_load_pat = r'\nmodule load %(mod)s'
+ else:
+ guarded_load_pat = r'if { \!\[ is-loaded %(mod)s \] } {\n\s*module load %(mod)s'
recursive_unload_pat = r'if { \[ module-info mode remove \] \|\| \!\[ is-loaded %(mod)s \] } {\n'
recursive_unload_pat += r'\s*module load %(mod)s'
@@ -4599,6 +4861,8 @@ def test_recursive_module_unload(self):
# recursive_module_unload easyconfig parameter is honored
test_ec_bis = os.path.join(self.test_prefix, 'test_bis.eb')
test_ec_bis_txt = read_file(toy_ec) + '\nrecursive_module_unload = True'
+ # this test only makes sense if depends_on is not used
+ test_ec_bis_txt += '\nmodule_depends_on = False'
write_file(test_ec_bis, test_ec_bis_txt)
ec_bis = EasyConfig(test_ec_bis)
@@ -4609,10 +4873,16 @@ def test_recursive_module_unload(self):
eb_bis.prepare_step()
eb_bis.make_module_step()
modtxt = read_file(test_module)
- fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt)
- self.assertFalse(guarded_load_regex.search(modtxt), fail_msg)
- fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt)
- self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg)
+ if self.modtool.supports_safe_auto_load:
+ fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt)
+ self.assertTrue(guarded_load_regex.search(modtxt), fail_msg)
+ fail_msg = "Pattern '%s' should not be found in: %s" % (recursive_unload_regex.pattern, modtxt)
+ self.assertFalse(recursive_unload_regex.search(modtxt), fail_msg)
+ else:
+ fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt)
+ self.assertFalse(guarded_load_regex.search(modtxt), fail_msg)
+ fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt)
+ self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg)
# recursive_mod_unload build option is honored
update_build_option('recursive_mod_unload', True)
@@ -4622,15 +4892,23 @@ def test_recursive_module_unload(self):
eb.prepare_step()
eb.make_module_step()
modtxt = read_file(test_module)
- fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt)
- self.assertFalse(guarded_load_regex.search(modtxt), fail_msg)
- fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt)
- self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg)
+ if self.modtool.supports_safe_auto_load:
+ fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt)
+ self.assertTrue(guarded_load_regex.search(modtxt), fail_msg)
+ fail_msg = "Pattern '%s' should not be found in: %s" % (recursive_unload_regex.pattern, modtxt)
+ self.assertFalse(recursive_unload_regex.search(modtxt), fail_msg)
+ else:
+ fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt)
+ self.assertFalse(guarded_load_regex.search(modtxt), fail_msg)
+ fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt)
+ self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg)
# disabling via easyconfig parameter works even when recursive_mod_unload build option is enabled
self.assertTrue(build_option('recursive_mod_unload'))
test_ec_bis = os.path.join(self.test_prefix, 'test_bis.eb')
test_ec_bis_txt = read_file(toy_ec) + '\nrecursive_module_unload = False'
+ # this test only makes sense if depends_on is not used
+ test_ec_bis_txt += '\nmodule_depends_on = False'
write_file(test_ec_bis, test_ec_bis_txt)
ec_bis = EasyConfig(test_ec_bis)
self.assertEqual(ec_bis['recursive_module_unload'], False)
@@ -4745,8 +5023,8 @@ def test_get_cuda_cc_template_value(self):
update_build_option('cuda_compute_capabilities', ['6.5', '7.0'])
ec = EasyConfig(self.eb_file)
- for key in cuda_template_values:
- self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key])
+ for key, expected in cuda_template_values.items():
+ self.assertEqual(ec.get_cuda_cc_template_value(key), expected)
update_build_option('cuda_compute_capabilities', None)
ec = EasyConfig(self.eb_file)
@@ -4758,8 +5036,8 @@ def test_get_cuda_cc_template_value(self):
self.prep()
ec = EasyConfig(self.eb_file)
- for key in cuda_template_values:
- self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key])
+ for key, expected in cuda_template_values.items():
+ self.assertEqual(ec.get_cuda_cc_template_value(key), expected)
def test_count_files(self):
"""Tests for EasyConfig.count_files method."""
diff --git a/test/framework/easyconfigformat.py b/test/framework/easyconfigformat.py
index e806a88378..7b84a2c867 100644
--- a/test/framework/easyconfigformat.py
+++ b/test/framework/easyconfigformat.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py
index 638137ec56..65d2084d55 100644
--- a/test/framework/easyconfigparser.py
+++ b/test/framework/easyconfigparser.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb
index 4ae349d0a0..1e3f34a777 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb
@@ -9,13 +9,10 @@ toolchain = SYSTEM
sources = [SOURCE_TAR_GZ]
checksums = [[
- 'be662daa971a640e40be5c804d9d7d10', # default (MD5)
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256)
- ('adler32', '0x998410035'),
- ('crc32', '0x1553842328'),
- ('md5', 'be662daa971a640e40be5c804d9d7d10'),
- ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
- ('size', 273),
+ ('sha512',
+ '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+ 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
]]
patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch']
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb
index ced8241a45..70a231fc8d 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb
@@ -10,13 +10,10 @@ toolchainopts = {'pic': True, 'opt': True, 'optarch': True}
sources = [SOURCE_TAR_GZ]
checksums = [[
- 'be662daa971a640e40be5c804d9d7d10', # default (MD5)
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256)
- ('adler32', '0x998410035'),
- ('crc32', '0x1553842328'),
- ('md5', 'be662daa971a640e40be5c804d9d7d10'),
- ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
- ('size', 273),
+ ('sha512',
+ '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+ 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
{SOURCE_TAR_GZ: '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',
'bar.tgz': '33ac60685a3e29538db5094259ea85c15906cbd0f74368733f4111eab6187c8f'},
]]
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb
index 925432d02a..c8d0504764 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a.eb
@@ -9,13 +9,10 @@ toolchainopts = {'pic': True, 'opt': True, 'optarch': True}
sources = [SOURCE_TAR_GZ]
checksums = [[
- 'be662daa971a640e40be5c804d9d7d10', # default (MD5)
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256)
- ('adler32', '0x998410035'),
- ('crc32', '0x1553842328'),
- ('md5', 'be662daa971a640e40be5c804d9d7d10'),
- ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
- ('size', 273),
+ ('sha512',
+ '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+ 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
]]
patches = [
'toy-0.0_fix-silly-typo-in-printf-statement.patch',
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb
index 68ece2259f..e02e21f7ae 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb
@@ -11,8 +11,8 @@ toolchain = SYSTEM
sources = [SOURCE_TAR_GZ]
patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch']
checksums = [
- ('adler32', '0x998410035'),
- 'a99f2a72cee1689a2f7e3ace0356efb1',
+ '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',
+ '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487',
]
moduleclass = 'tools'
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb
index 90cc7429d3..286cdf7f6c 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb
@@ -9,13 +9,10 @@ toolchain = SYSTEM
sources = [SOURCE_TAR_GZ]
checksums = [[
- 'be662daa971a640e40be5c804d9d7d10', # default (MD5)
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256)
- ('adler32', '0x998410035'),
- ('crc32', '0x1553842328'),
- ('md5', 'be662daa971a640e40be5c804d9d7d10'),
- ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
- ('size', 273),
+ ('sha512',
+ '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+ 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
]]
patches = [
'toy-0.0_fix-silly-typo-in-printf-statement.patch',
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb
index c2a88616b1..0e087cec7c 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb
@@ -8,13 +8,10 @@ toolchain = SYSTEM
sources = [SOURCE_TAR_GZ]
checksums = [[
- 'be662daa971a640e40be5c804d9d7d10', # default (MD5)
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256)
- ('adler32', '0x998410035'),
- ('crc32', '0x1553842328'),
- ('md5', 'be662daa971a640e40be5c804d9d7d10'),
- ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
- ('size', 273),
+ ('sha512',
+ '3c9dc629e1f2fd01a15c68f9f2a328b5da045c2ec1a189dc72d7195642f32e0'
+ 'ff59275aba5fa2a78e84417c7645d0ca5d06aff39e688a8936061ed5c4c600708'),
]]
patches = [
'toy-0.0_fix-silly-typo-in-printf-statement.patch',
diff --git a/test/framework/easyconfigs/v2.0/toy-with-sections.eb b/test/framework/easyconfigs/v2.0/toy-with-sections.eb
index 34b9af0dcd..a1a508bcf0 100644
--- a/test/framework/easyconfigs/v2.0/toy-with-sections.eb
+++ b/test/framework/easyconfigs/v2.0/toy-with-sections.eb
@@ -16,7 +16,6 @@ software_license_urls = ['https://github.com/easybuilders/easybuild/wiki/License
sources = ['%(name)s-0.0.tar.gz'] # purposely fixed to 0.0
checksums = [
- 'be662daa971a640e40be5c804d9d7d10', # MD5
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # SHA256
]
diff --git a/test/framework/easyconfigs/v2.0/toy.eb b/test/framework/easyconfigs/v2.0/toy.eb
index a1cdfaf6d8..3c5a000f1b 100644
--- a/test/framework/easyconfigs/v2.0/toy.eb
+++ b/test/framework/easyconfigs/v2.0/toy.eb
@@ -16,7 +16,6 @@ software_license_urls = ['https://github.com/easybuilders/easybuild/wiki/License
sources = ['%(name)s-0.0.tar.gz'] # purposely fixed to 0.0
checksums = [
- 'be662daa971a640e40be5c804d9d7d10', # MD5
'44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # SHA256
]
diff --git a/test/framework/easyconfigversion.py b/test/framework/easyconfigversion.py
index 5ef706a72a..4da54b450a 100644
--- a/test/framework/easyconfigversion.py
+++ b/test/framework/easyconfigversion.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/easystack.py b/test/framework/easystack.py
index 198350a0e7..0c1067a34b 100644
--- a/test/framework/easystack.py
+++ b/test/framework/easystack.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/ebconfigobj.py b/test/framework/ebconfigobj.py
index d4d81995d8..6ae2caa4f9 100644
--- a/test/framework/ebconfigobj.py
+++ b/test/framework/ebconfigobj.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -116,6 +116,23 @@ def test_squash_simple(self):
res = cov.squash(version, tc['name'], tc['version'])
self.assertEqual(res, {}) # very simple
+ # Ensure that a version of '0' with trailing '.0's is matched against '0.0' but not anything higher
+ # This is for testing the DEFAULT_UNDEFINED_VERSION detection
+ for num_zeroes in range(1, 6):
+ tc = tc_first
+ zero_version = '.'.join(['0'] * num_zeroes)
+ txt = [
+ '[SUPPORTED]',
+ 'versions = ' + zero_version,
+ 'toolchains = ' + tc_tmpl % tc,
+ '[DEFAULT]',
+ 'y=a',
+ ]
+ co = ConfigObj(txt)
+ cov = EBConfigObj(co)
+ self.assertEqual(cov.squash('0.0', tc['name'], tc['version']), {'y': 'a'})
+ self.assertEqual(cov.squash('0.1', tc['name'], tc['version']), {})
+
def test_squash_invalid(self):
"""Try to squash invalid files. Should trigger error"""
tc_first = {'version': '10', 'name': self.tc_first}
@@ -123,8 +140,8 @@ def test_squash_invalid(self):
tc_tmpl = '%(name)s == %(version)s'
- default_version = '1.0'
- all_wrong_versions = [default_version, '>= 0.0', '< 1.0']
+ default_version = '1.1'
+ all_wrong_versions = [default_version, '>= 0.0', '< 1.1']
# all txt should have default version and first toolchain unmodified
diff --git a/test/framework/environment.py b/test/framework/environment.py
index 9a81e17486..f184c6b9c7 100644
--- a/test/framework/environment.py
+++ b/test/framework/environment.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -90,8 +90,7 @@ def test_modify_env(self):
# prepare test environment first:
# keys in new_env should not be set yet, keys in old_env are expected to be set
for key in new_env_vars:
- if key in os.environ:
- del os.environ[key]
+ os.environ.pop(key, None)
for key in old_env_vars:
os.environ[key] = old_env_vars[key]
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index bdd3f703de..911dc858ce 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -32,7 +32,9 @@
@author: Maxime Boissonneault (Compute Canada, Universite Laval)
"""
import datetime
+import filecmp
import glob
+import logging
import os
import re
import shutil
@@ -47,11 +49,12 @@
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from unittest import TextTestRunner
from urllib import request
-from easybuild.tools import run
import easybuild.tools.filetools as ft
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option
+from easybuild.tools.config import IGNORE, ERROR, WARN, build_option, update_build_option
from easybuild.tools.multidiff import multidiff
+from easybuild.tools.run import run_shell_cmd
+from easybuild.tools.systemtools import LINUX, get_os_type
class FileToolsTest(EnhancedTestCase):
@@ -290,29 +293,39 @@ def test_checksums(self):
ft.write_file(fp, "easybuild\n")
known_checksums = {
- 'adler32': '0x379257805',
- 'crc32': '0x1457143216',
- 'md5': '7167b64b1ca062b9674ffef46f9325db',
- 'sha1': 'db05b79e09a4cc67e9dd30b313b5488813db3190',
'sha256': '1c49562c4b404f3120a3fa0926c8d09c99ef80e470f7de03ffdfa14047960ea5',
'sha512': '7610f6ce5e91e56e350d25c917490e4815f7986469fafa41056698aec256733e'
'b7297da8b547d5e74b851d7c4e475900cec4744df0f887ae5c05bf1757c224b4',
}
+ old_log_level = ft._log.getEffectiveLevel()
+ ft._log.setLevel(logging.DEBUG)
# make sure checksums computation/verification is correct
for checksum_type, checksum in known_checksums.items():
self.assertEqual(ft.compute_checksum(fp, checksum_type=checksum_type), checksum)
- self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum)))
-
- # default checksum type is MD5
- self.assertEqual(ft.compute_checksum(fp), known_checksums['md5'])
-
- # both MD5 and SHA256 checksums can be verified without specifying type
- self.assertTrue(ft.verify_checksum(fp, known_checksums['md5']))
+ with self.log_to_testlogfile():
+ self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum)))
+ self.assertIn('Computed ' + checksum_type, ft.read_file(self.logfile))
+ # Passing precomputed checksums reuses it
+ with self.log_to_testlogfile():
+ computed_checksums = {checksum_type: checksum}
+ self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum), computed_checksums))
+ self.assertIn('Precomputed ' + checksum_type, ft.read_file(self.logfile))
+ # If the type isn't contained the checksum will be computed
+ with self.log_to_testlogfile():
+ computed_checksums = {'doesnt exist': 'checksum'}
+ self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum), computed_checksums))
+ self.assertIn('Computed ' + checksum_type, ft.read_file(self.logfile))
+
+ ft._log.setLevel(old_log_level)
+
+ # default checksum type is SHA256
+ self.assertEqual(ft.compute_checksum(fp), known_checksums['sha256'])
+
+ # SHA256 checksums can be verified without specifying type
self.assertTrue(ft.verify_checksum(fp, known_checksums['sha256']))
- # providing non-matching MD5 and SHA256 checksums results in failed verification
- self.assertFalse(ft.verify_checksum(fp, '1c49562c4b404f3120a3fa0926c8d09c'))
+ # providing non-matching SHA256 checksums results in failed verification
self.assertFalse(ft.verify_checksum(fp, '7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db'))
# checksum of length 32 is assumed to be MD5, length 64 to be SHA256, other lengths not allowed
@@ -322,29 +335,18 @@ def test_checksums(self):
self.assertErrorRegex(EasyBuildError, error_pattern, ft.verify_checksum, fp, checksum)
# make sure faulty checksums are reported
- broken_checksums = dict([(typ, val[:-3] + 'foo') for (typ, val) in known_checksums.items()])
+ broken_checksums = {typ: (val[:-3] + 'foo') for typ, val in known_checksums.items()}
for checksum_type, checksum in broken_checksums.items():
self.assertFalse(ft.compute_checksum(fp, checksum_type=checksum_type) == checksum)
self.assertFalse(ft.verify_checksum(fp, (checksum_type, checksum)))
- # md5 is default
- self.assertFalse(ft.compute_checksum(fp) == broken_checksums['md5'])
- self.assertFalse(ft.verify_checksum(fp, broken_checksums['md5']))
+ # sha256 is default
+ self.assertFalse(ft.compute_checksum(fp) == broken_checksums['sha256'])
self.assertFalse(ft.verify_checksum(fp, broken_checksums['sha256']))
# test specify alternative checksums
alt_checksums = ('7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db', known_checksums['sha256'])
self.assertTrue(ft.verify_checksum(fp, alt_checksums))
- alt_checksums = ('fecf50db81148786647312bbd3b5c740', '2c829facaba19c0fcd81f9ce96bef712',
- '840078aeb4b5d69506e7c8edae1e1b89', known_checksums['md5'])
- self.assertTrue(ft.verify_checksum(fp, alt_checksums))
-
- alt_checksums = ('840078aeb4b5d69506e7c8edae1e1b89', known_checksums['md5'], '2c829facaba19c0fcd81f9ce96bef712')
- self.assertTrue(ft.verify_checksum(fp, alt_checksums))
-
- alt_checksums = (known_checksums['md5'], '840078aeb4b5d69506e7c8edae1e1b89', '2c829facaba19c0fcd81f9ce96bef712')
- self.assertTrue(ft.verify_checksum(fp, alt_checksums))
-
alt_checksums = (known_checksums['sha256'],)
self.assertTrue(ft.verify_checksum(fp, alt_checksums))
@@ -366,16 +368,84 @@ def test_checksums(self):
init_config(build_options=build_options)
self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, None)
- self.assertTrue(ft.verify_checksum(fp, known_checksums['md5']))
self.assertTrue(ft.verify_checksum(fp, known_checksums['sha256']))
# Test dictionary-type checksums
- for checksum in [known_checksums[x] for x in ('md5', 'sha256')]:
+ for checksum in [known_checksums[x] for x in ['sha256']]:
+ dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'}
+ self.assertTrue(ft.verify_checksum(fp, dict_checksum))
+ del dict_checksum[os.path.basename(fp)]
+ self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum)
+
+ def test_deprecated_checksums(self):
+ """Test checksum functionality."""
+
+ fp = os.path.join(self.test_prefix, 'test.txt')
+ ft.write_file(fp, "easybuild\n")
+
+ known_checksums = {
+ 'adler32': '0x379257805',
+ 'crc32': '0x1457143216',
+ 'md5': '7167b64b1ca062b9674ffef46f9325db',
+ 'sha1': 'db05b79e09a4cc67e9dd30b313b5488813db3190',
+ }
+
+ self.allow_deprecated_behaviour()
+ self.mock_stderr(True) # just to capture deprecation warning
+
+ # make sure checksums computation/verification is correct
+ for checksum_type, checksum in known_checksums.items():
+ self.assertEqual(ft.compute_checksum(fp, checksum_type=checksum_type), checksum)
+ self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum)))
+
+ # MD5 checksums can be verified without specifying type
+ self.assertTrue(ft.verify_checksum(fp, known_checksums['md5']))
+
+ # providing non-matching MD5 checksums results in failed verification
+ self.assertFalse(ft.verify_checksum(fp, '1c49562c4b404f3120a3fa0926c8d09c'))
+
+ # checksum of length 32 is assumed to be MD5, length 64 to be SHA256, other lengths not allowed
+ # checksum of length other than 32/64 yields an error
+ error_pattern = r"Length of checksum '.*' \(\d+\) does not match with either MD5 \(32\) or SHA256 \(64\)"
+ for checksum in ['tooshort', 'inbetween32and64charactersisnotgoodeither', known_checksums['md5'] + 'foo']:
+ self.assertErrorRegex(EasyBuildError, error_pattern, ft.verify_checksum, fp, checksum)
+
+ # make sure faulty checksums are reported
+ broken_checksums = {typ: (val[:-3] + 'foo') for typ, val in known_checksums.items()}
+ for checksum_type, checksum in broken_checksums.items():
+ self.assertFalse(ft.compute_checksum(fp, checksum_type=checksum_type) == checksum)
+ self.assertFalse(ft.verify_checksum(fp, (checksum_type, checksum)))
+ self.assertFalse(ft.verify_checksum(fp, broken_checksums['md5']))
+
+ # test specify alternative checksums
+ alt_checksums = ('fecf50db81148786647312bbd3b5c740', '2c829facaba19c0fcd81f9ce96bef712',
+ '840078aeb4b5d69506e7c8edae1e1b89', known_checksums['md5'])
+ self.assertTrue(ft.verify_checksum(fp, alt_checksums))
+
+ alt_checksums = ('840078aeb4b5d69506e7c8edae1e1b89', known_checksums['md5'], '2c829facaba19c0fcd81f9ce96bef712')
+ self.assertTrue(ft.verify_checksum(fp, alt_checksums))
+
+ alt_checksums = (known_checksums['md5'], '840078aeb4b5d69506e7c8edae1e1b89', '2c829facaba19c0fcd81f9ce96bef712')
+ self.assertTrue(ft.verify_checksum(fp, alt_checksums))
+
+ # check whether missing checksums are enforced
+ build_options = {
+ 'enforce_checksums': True,
+ }
+ init_config(build_options=build_options)
+
+ self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, None)
+ self.assertTrue(ft.verify_checksum(fp, known_checksums['md5']))
+
+ # Test dictionary-type checksums
+ for checksum in [known_checksums[x] for x in ['md5']]:
dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'}
self.assertTrue(ft.verify_checksum(fp, dict_checksum))
del dict_checksum[os.path.basename(fp)]
self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum)
+ self.mock_stderr(False)
+
def test_common_path_prefix(self):
"""Test get common path prefix for a list of paths."""
self.assertEqual(ft.det_common_path_prefix(['/foo/bar/foo', '/foo/bar/baz', '/foo/bar/bar']), '/foo/bar')
@@ -924,7 +994,7 @@ def test_is_binary(self):
self.assertTrue(ft.is_binary(b'\00'))
self.assertTrue(ft.is_binary(b"File is binary when it includes \00 somewhere"))
- self.assertTrue(ft.is_binary(ft.read_file('/bin/ls', mode='rb')))
+ self.assertTrue(ft.is_binary(ft.read_file('/bin/bash', mode='rb')))
def test_det_patched_files(self):
"""Test det_patched_files function."""
@@ -1163,9 +1233,9 @@ def test_multidiff(self):
self.assertTrue(lines[8].startswith(expected))
# no postinstallcmds in toy-0.0-deps.eb
- expected = "29 %s+ postinstallcmds = " % green
+ expected = "26 %s+ postinstallcmds = " % green
self.assertTrue(any(line.startswith(expected) for line in lines))
- expected = "30 %s+%s (1/2) toy-0.0" % (green, endcol)
+ expected = "27 %s+%s (1/2) toy-0.0" % (green, endcol)
self.assertTrue(any(line.startswith(expected) for line in lines), "Found '%s' in: %s" % (expected, lines))
self.assertEqual(lines[-1], "=====")
@@ -1184,9 +1254,9 @@ def test_multidiff(self):
self.assertTrue(lines[8].startswith(expected))
# no postinstallcmds in toy-0.0-deps.eb
- expected = "29 + postinstallcmds = "
+ expected = "26 + postinstallcmds = "
self.assertTrue(any(line.startswith(expected) for line in lines), "Found '%s' in: %s" % (expected, lines))
- expected = "30 + (1/2) toy-0.0-"
+ expected = "27 + (1/2) toy-0.0-"
self.assertTrue(any(line.startswith(expected) for line in lines), "Found '%s' in: %s" % (expected, lines))
self.assertEqual(lines[-1], "=====")
@@ -1442,7 +1512,7 @@ def test_apply_regex_substitutions(self):
# passing empty list of substitions is a no-op
ft.write_file(testfile, testtxt)
- ft.apply_regex_substitutions(testfile, [], on_missing_match=run.IGNORE)
+ ft.apply_regex_substitutions(testfile, [], on_missing_match=IGNORE)
new_testtxt = ft.read_file(testfile)
self.assertEqual(new_testtxt, testtxt)
@@ -1452,17 +1522,17 @@ def test_apply_regex_substitutions(self):
error_pat = 'Nothing found to replace in %s' % testfile
# Error
self.assertErrorRegex(EasyBuildError, error_pat, ft.apply_regex_substitutions, testfile, regex_subs_no_match,
- on_missing_match=run.ERROR)
+ on_missing_match=ERROR)
# Warn
with self.log_to_testlogfile():
- ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.WARN)
+ ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=WARN)
logtxt = ft.read_file(self.logfile)
self.assertIn('WARNING ' + error_pat, logtxt)
# Ignore
with self.log_to_testlogfile():
- ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=run.IGNORE)
+ ft.apply_regex_substitutions(testfile, regex_subs_no_match, on_missing_match=IGNORE)
logtxt = ft.read_file(self.logfile)
self.assertIn('INFO ' + error_pat, logtxt)
@@ -1531,8 +1601,7 @@ def test_find_flexlm_license(self):
lic_server = '1234@example.license.server'
# make test robust against environment in which $LM_LICENSE_FILE is defined
- if 'LM_LICENSE_FILE' in os.environ:
- del os.environ['LM_LICENSE_FILE']
+ os.environ.pop('LM_LICENSE_FILE', None)
# default return value
self.assertEqual(ft.find_flexlm_license(), ([], None))
@@ -1854,6 +1923,19 @@ def test_copy_file(self):
# printing this message will make test suite fail in Travis/GitHub CI,
# since we check for unexpected output produced by the tests
print("Skipping overwrite-file-owned-by-other-user copy_file test (%s is missing)" % test_file_to_overwrite)
+ # Copy a file to a directory owned by some other user, e.g. /tmp (owned by root)
+ # This might be a common choice for e.g. --copy-ec
+ target_file_path = tempfile.mktemp("easybuild", dir="/tmp")
+ test_file_to_copy = os.path.join(self.test_prefix, os.path.basename(target_file_path))
+ ft.write_file(test_file_to_copy, test_file_contents)
+ try:
+ ft.copy_file(test_file_to_copy, '/tmp')
+ self.assertEqual(ft.read_file(target_file_path), test_file_contents)
+ finally:
+ try:
+ os.remove(target_file_path)
+ except FileNotFoundError:
+ pass
# also test behaviour of copy_file under --dry-run
build_options = {
@@ -1897,6 +1979,49 @@ def test_copy_file(self):
# However, if we add 'force_in_dry_run=True' it should throw an exception
self.assertErrorRegex(EasyBuildError, "Could not copy *", ft.copy_file, src, target, force_in_dry_run=True)
+ def test_copy_file_xattr(self):
+ """Test copying a file with extended attributes using copy_file."""
+ # test copying a read-only files with extended attributes set
+ # first, create a special file with extended attributes
+ special_file = os.path.join(self.test_prefix, 'special.txt')
+ ft.write_file(special_file, 'special')
+ # make read-only, and set extended attributes
+ attr = ft.which('attr')
+ xattr = ft.which('xattr')
+ # try to attr (Linux) or xattr (macOS) to set extended attributes foo=bar
+ cmd = None
+ if attr:
+ cmd = "attr -s foo -V bar %s" % special_file
+ elif xattr:
+ cmd = "xattr -w foo bar %s" % special_file
+
+ if cmd:
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, fail_on_error=False)
+
+ # need to make file read-only after setting extended attribute
+ ft.adjust_permissions(special_file, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False)
+
+ # only proceed if setting extended attribute worked
+ if res.exit_code == 0:
+ target = os.path.join(self.test_prefix, 'copy.txt')
+ ft.copy_file(special_file, target)
+ self.assertTrue(os.path.exists(target))
+ self.assertTrue(filecmp.cmp(special_file, target, shallow=False))
+
+ # only verify wheter extended attributes were also copied on Linux,
+ # since shutil.copy2 doesn't copy them on macOS;
+ # see warning at https://docs.python.org/3/library/shutil.html
+ if get_os_type() == LINUX:
+ if attr:
+ cmd = "attr -g foo %s" % target
+ else:
+ cmd = "xattr -l %s" % target
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, fail_on_error=False)
+ self.assertEqual(res.exit_code, 0)
+ self.assertTrue(res.output.endswith('\nbar\n'))
+
def test_copy_files(self):
"""Test copy_files function."""
test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
@@ -2224,6 +2349,19 @@ def test_copy(self):
self.assertTrue(os.path.isfile(os.path.join(self.test_prefix, 'GCC-4.6.3.eb')))
self.assertEqual(txt, '')
+ def test_get_cwd(self):
+ """Test get_cwd"""
+ toy_dir = os.path.join(self.test_prefix, "test_get_cwd_dir")
+ os.mkdir(toy_dir)
+ os.chdir(toy_dir)
+
+ self.assertTrue(os.path.samefile(ft.get_cwd(), toy_dir))
+
+ os.rmdir(toy_dir)
+ self.assertErrorRegex(EasyBuildError, ft.CWD_NOTFOUND_ERROR, ft.get_cwd)
+
+ self.assertEqual(ft.get_cwd(must_exist=False), None)
+
def test_change_dir(self):
"""Test change_dir"""
@@ -2637,8 +2775,7 @@ def test_find_eb_script(self):
"""Test find_eb_script function."""
# make sure $EB_SCRIPT_PATH is not set already (used as fallback mechanism in find_eb_script)
- if 'EB_SCRIPT_PATH' in os.environ:
- del os.environ['EB_SCRIPT_PATH']
+ os.environ.pop('EB_SCRIPT_PATH', None)
self.assertExists(ft.find_eb_script('rpath_args.py'))
self.assertExists(ft.find_eb_script('rpath_wrapper_template.sh.in'))
@@ -2798,32 +2935,41 @@ def run_check():
'url': 'git@github.com:easybuilders',
'tag': 'tag_for_tests',
}
- git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter
+ string_args = {
+ 'git_repo': 'git@github.com:easybuilders/testrepository.git',
+ 'test_prefix': self.test_prefix,
+ }
+ reprod_tar_cmd_pattern = (
+ r' running shell command "find {} -name \".git\" -prune -o -print0 -exec touch -t 197001010100 {{}} \; |'
+ r' LC_ALL=C sort --zero-terminated | tar --create --no-recursion --owner=0 --group=0 --numeric-owner'
+ r' --format=gnu --null --files-from - | gzip --no-name > %(test_prefix)s/target/test.tar.gz'
+ )
+
expected = '\n'.join([
r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"',
- r" \(in /.*\)",
- r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
- r" \(in /.*\)",
- ]) % git_repo
+ r" \(in .*/tmp.*\)",
+ reprod_tar_cmd_pattern.format("testrepository"),
+ r" \(in .*/tmp.*\)",
+ ]) % string_args
run_check()
git_config['clone_into'] = 'test123'
expected = '\n'.join([
r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"',
- r" \(in /.*\)",
- r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git test123"',
- r" \(in /.*\)",
- ]) % git_repo
+ r" \(in .*/tmp.*\)",
+ reprod_tar_cmd_pattern.format("test123"),
+ r" \(in .*/tmp.*\)",
+ ]) % string_args
run_check()
del git_config['clone_into']
git_config['recursive'] = True
expected = '\n'.join([
r' running shell command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"',
- r" \(in /.*\)",
- r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
- r" \(in /.*\)",
- ]) % git_repo
+ r" \(in .*/tmp.*\)",
+ reprod_tar_cmd_pattern.format("testrepository"),
+ r" \(in .*/tmp.*\)",
+ ]) % string_args
run_check()
git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite']
@@ -2831,9 +2977,9 @@ def run_check():
' running shell command "git clone --depth 1 --branch tag_for_tests --recursive'
+ ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"',
r" \(in .*/tmp.*\)",
- r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+ reprod_tar_cmd_pattern.format("testrepository"),
r" \(in .*/tmp.*\)",
- ]) % git_repo
+ ]) % string_args
run_check()
git_config['extra_config_params'] = [
@@ -2845,9 +2991,9 @@ def run_check():
+ ' clone --depth 1 --branch tag_for_tests --recursive'
+ ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"',
r" \(in .*/tmp.*\)",
- r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+ reprod_tar_cmd_pattern.format("testrepository"),
r" \(in .*/tmp.*\)",
- ]) % git_repo
+ ]) % string_args
run_check()
del git_config['recurse_submodules']
del git_config['extra_config_params']
@@ -2855,10 +3001,10 @@ def run_check():
git_config['keep_git_dir'] = True
expected = '\n'.join([
r' running shell command "git clone --branch tag_for_tests --recursive %(git_repo)s"',
- r" \(in /.*\)",
+ r" \(in .*/tmp.*\)",
r' running shell command "tar cfvz .*/target/test.tar.gz testrepository"',
- r" \(in /.*\)",
- ]) % git_repo
+ r" \(in .*/tmp.*\)",
+ ]) % string_args
run_check()
del git_config['keep_git_dir']
@@ -2866,24 +3012,23 @@ def run_check():
git_config['commit'] = '8456f86'
expected = '\n'.join([
r' running shell command "git clone --no-checkout %(git_repo)s"',
- r" \(in /.*\)",
+ r" \(in .*/tmp.*\)",
r' running shell command "git checkout 8456f86 && git submodule update --init --recursive"',
- r" \(in /.*/testrepository\)",
- r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
- r" \(in /.*\)",
- ]) % git_repo
+ r" \(in testrepository\)",
+ reprod_tar_cmd_pattern.format("testrepository"),
+ r" \(in .*/tmp.*\)",
+ ]) % string_args
run_check()
git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite']
expected = '\n'.join([
r' running shell command "git clone --no-checkout %(git_repo)s"',
r" \(in .*/tmp.*\)",
- ' running shell command "git checkout 8456f86 && git submodule update --init --recursive'
- + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\'"',
- r" \(in /.*/testrepository\)",
- r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+ r' running shell command "git checkout 8456f86"',
+ r" \(in testrepository\)",
+ reprod_tar_cmd_pattern.format("testrepository"),
r" \(in .*/tmp.*\)",
- ]) % git_repo
+ ]) % string_args
run_check()
del git_config['recursive']
@@ -2893,9 +3038,9 @@ def run_check():
r" \(in /.*\)",
r' running shell command "git checkout 8456f86"',
r" \(in /.*/testrepository\)",
- r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+ reprod_tar_cmd_pattern.format("testrepository"),
r" \(in /.*\)",
- ]) % git_repo
+ ]) % string_args
run_check()
# Test with real data.
diff --git a/test/framework/format_convert.py b/test/framework/format_convert.py
index 1bc3f764bb..e0add188b3 100644
--- a/test/framework/format_convert.py
+++ b/test/framework/format_convert.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/general.py b/test/framework/general.py
index 526fe07467..3837cecc87 100644
--- a/test/framework/general.py
+++ b/test/framework/general.py
@@ -1,5 +1,5 @@
##
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/github.py b/test/framework/github.py
index 890ccfa28a..21461fb430 100644
--- a/test/framework/github.py
+++ b/test/framework/github.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -52,7 +52,8 @@
from easybuild.tools.filetools import read_file, write_file
from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_MERGEABLE_STATE_CLEAN
from easybuild.tools.github import VALID_CLOSE_PR_REASONS
-from easybuild.tools.github import det_pr_title, is_patch_for, pick_default_branch
+from easybuild.tools.github import det_pr_title, fetch_easyconfigs_from_commit, fetch_files_from_commit
+from easybuild.tools.github import is_patch_for, pick_default_branch
from easybuild.tools.testing import create_test_report, post_pr_test_report, session_state
import easybuild.tools.github as gh
@@ -541,6 +542,65 @@ def test_github_fetch_files_from_pr_cache(self):
res = gh.fetch_easyblocks_from_pr(12345, tmpdir)
self.assertEqual(sorted(pr12345_files), sorted(res))
+ def test_fetch_files_from_commit(self):
+ """Test fetch_files_from_commit function."""
+
+ # easyconfigs commit to add EasyBuild-4.8.2.eb
+ test_commit = '7c83a553950c233943c7b0189762f8c05cfea852'
+
+ # without specifying any files/repo, default is to use easybuilders/easybuilld-easyconfigs
+ # and determine which files were changed in the commit
+ res = fetch_files_from_commit(test_commit)
+ self.assertEqual(len(res), 1)
+ ec_path = res[0]
+ expected_path = 'ecs_commit_7c83a553950c233943c7b0189762f8c05cfea852/e/EasyBuild/EasyBuild-4.8.2.eb'
+ self.assertTrue(ec_path.endswith(expected_path))
+ self.assertTrue(os.path.exists(ec_path))
+ self.assertIn("version = '4.8.2'", read_file(ec_path))
+
+ # also test downloading a specific file from easyblocks repo
+ # commit that enables use_pip & co in PythonPackage easyblock
+ test_commit = 'd6f0cd7b586108e40f7cf1f1054bb07e16718caf'
+ res = fetch_files_from_commit(test_commit, files=['pythonpackage.py'],
+ github_account='easybuilders', github_repo='easybuild-easyblocks')
+ self.assertEqual(len(res), 1)
+ self.assertIn("'use_pip': [True,", read_file(res[0]))
+
+ # test downloading with short commit, download_repo currently enforces using long commit
+ error_pattern = r"Specified commit SHA 7c83a55 for downloading easybuilders/easybuild-easyconfigs "
+ error_pattern += r"is not valid, must be full SHA-1 \(40 chars\)"
+ self.assertErrorRegex(EasyBuildError, error_pattern, fetch_files_from_commit, '7c83a55')
+
+ # test downloading of non-existing commit
+ error_pattern = r"Failed to download diff for commit c0ff33c0ff33 of easybuilders/easybuild-easyconfigs"
+ self.assertErrorRegex(EasyBuildError, error_pattern, fetch_files_from_commit, 'c0ff33c0ff33')
+
+ def test_fetch_easyconfigs_from_commit(self):
+ """Test fetch_easyconfigs_from_commit function."""
+
+ # commit in which easyconfigs for PyTables 3.9.2 + dependencies were added
+ test_commit = '6515b44cd84a20fe7876cb4bdaf3c0080e688566'
+
+ # without specifying any files/repo, default is to determine which files were changed in the commit
+ res = fetch_easyconfigs_from_commit(test_commit)
+ self.assertEqual(len(res), 5)
+ expected_ec_filenames = ['Blosc-1.21.5-GCCcore-13.2.0.eb', 'Blosc2-2.13.2-GCCcore-13.2.0.eb',
+ 'PyTables-3.9.2-foss-2023b.eb', 'PyTables-3.9.2_fix-find-blosc2-dep.patch',
+ 'py-cpuinfo-9.0.0-GCCcore-13.2.0.eb']
+ self.assertEqual(sorted([os.path.basename(f) for f in res]), expected_ec_filenames)
+ for ec_path in res:
+ self.assertTrue(os.path.exists(ec_path))
+ if ec_path.endswith('.eb'):
+ self.assertIn("version =", read_file(ec_path))
+ else:
+ self.assertTrue(ec_path.endswith('.patch'))
+
+ # merge commit for release of EasyBuild v4.9.0
+ test_commit = 'bdcc586189fcb3e5a340cddebb50d0e188c63cdc'
+ res = fetch_easyconfigs_from_commit(test_commit, files=['RELEASE_NOTES'], path=self.test_prefix)
+ self.assertEqual(len(res), 1)
+ self.assertIn("v4.9.0 (30 December 2023)", read_file(res[0]))
+
def test_github_fetch_latest_commit_sha(self):
"""Test fetch_latest_commit_sha function."""
if self.skip_github_tests:
@@ -597,6 +657,34 @@ def test_github_download_repo(self):
self.assertExists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py'))
self.mock_stdout(False)
+ def test_github_download_repo_commit(self):
+ """Test downloading repo at specific commit (which does not require any GitHub token)"""
+
+ # commit bdcc586189fcb3e5a340cddebb50d0e188c63cdc corresponds to easybuild-easyconfigs release v4.9.0
+ test_commit = 'bdcc586189fcb3e5a340cddebb50d0e188c63cdc'
+ gh.download_repo(path=self.test_prefix, commit=test_commit)
+ repo_path = os.path.join(self.test_prefix, 'easybuilders', 'easybuild-easyconfigs-' + test_commit)
+ self.assertTrue(os.path.exists(repo_path))
+
+ setup_py_txt = read_file(os.path.join(repo_path, 'setup.py'))
+ self.assertTrue("VERSION = '4.9.0'" in setup_py_txt)
+
+ # also check downloading non-default forked repo
+ test_commit = '434151c3dbf88b2382e8ead8655b4b2c01b92617'
+ gh.download_repo(path=self.test_prefix, account='boegel', repo='easybuild-framework', commit=test_commit)
+ repo_path = os.path.join(self.test_prefix, 'boegel', 'easybuild-framework-' + test_commit)
+ self.assertTrue(os.path.exists(repo_path))
+
+ release_notes_txt = read_file(os.path.join(repo_path, 'RELEASE_NOTES'))
+ self.assertTrue("v4.9.0 (30 December 2023)" in release_notes_txt)
+
+ # short commit doesn't work, must be full commit ID
+ self.assertErrorRegex(EasyBuildError, "Specified commit SHA bdcc586 .* is not valid", gh.download_repo,
+ path=self.test_prefix, commit='bdcc586')
+
+ self.assertErrorRegex(EasyBuildError, "Failed to download tarball .* commit", gh.download_repo,
+ path=self.test_prefix, commit='0000000000000000000000000000000000000000')
+
def test_install_github_token(self):
"""Test for install_github_token function."""
if self.skip_github_tests:
@@ -738,18 +826,21 @@ def test_github_det_commit_status(self):
res = gh.det_commit_status('easybuilders', GITHUB_REPO, commit_sha, GITHUB_TEST_ACCOUNT)
self.assertEqual(res, None)
- # recent commit (2023-04-11) with cancelled checks (GitHub Actions only)
- commit_sha = 'c074f0bb3110c27d9969c3d0b19dde3eca868bd4'
+ # recent commit with cancelled checks (GitHub Actions only);
+ # to update, use https://github.com/easybuilders/easybuild-easyconfigs/actions?query=is%3Acancelled
+ commit_sha = '52b964c3387d6d6f149ec304f9e23f535e799957'
res = gh.det_commit_status('easybuilders', 'easybuild-easyconfigs', commit_sha, GITHUB_TEST_ACCOUNT)
self.assertEqual(res, 'cancelled')
- # recent commit (2023-04-10) with failing checks (GitHub Actions only)
- commit_sha = '1b4a45c62d7deaf19125756c46dc8f011fef66e1'
+ # recent commit with failing checks (GitHub Actions only)
+ # to update, use https://github.com/easybuilders/easybuild-easyconfigs/actions?query=is%3Afailure
+ commit_sha = '85e6c2bbc2fd515a1d4dab607b8d43d0a1ed668f'
res = gh.det_commit_status('easybuilders', 'easybuild-easyconfigs', commit_sha, GITHUB_TEST_ACCOUNT)
self.assertEqual(res, 'failure')
- # recent commit (2023-04-10) with successful checks (GitHub Actions only)
- commit_sha = '56812a347acbaaa87f229fe319425020fe399647'
+ # recent commit with successful checks (GitHub Actions only)
+ # to update, use https://github.com/easybuilders/easybuild-easyconfigs/actions?query=is%3Asuccess
+ commit_sha = 'f82a563b8e1f8118c7c3ab23374d0e28e1691fea'
res = gh.det_commit_status('easybuilders', 'easybuild-easyconfigs', commit_sha, GITHUB_TEST_ACCOUNT)
self.assertEqual(res, 'success')
diff --git a/test/framework/hooks.py b/test/framework/hooks.py
index c8e5d34583..0cc478f53e 100644
--- a/test/framework/hooks.py
+++ b/test/framework/hooks.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2017-2023 Ghent University
+# Copyright 2017-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/include.py b/test/framework/include.py
index b346f911ae..09a5925a2f 100644
--- a/test/framework/include.py
+++ b/test/framework/include.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/lib.py b/test/framework/lib.py
index a6459b28c3..bcc4c8e7b4 100644
--- a/test/framework/lib.py
+++ b/test/framework/lib.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2018-2023 Ghent University
+# Copyright 2018-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -74,14 +74,14 @@ def test_run_cmd(self):
error_pattern = r"Undefined build option: .*"
error_pattern += r" Make sure you have set up the EasyBuild configuration using set_up_configuration\(\)"
- self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd, "echo hello")
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd, "echo hello")
self.configure()
# run_cmd works fine if set_up_configuration was called first
- self.mock_stdout(True)
- (out, ec) = run_cmd("echo hello")
- self.mock_stdout(False)
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd("echo hello")
self.assertEqual(ec, 0)
self.assertEqual(out, 'hello\n')
diff --git a/test/framework/license.py b/test/framework/license.py
index 8b231a8346..768c5ed6b4 100644
--- a/test/framework/license.py
+++ b/test/framework/license.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py
index d2fbb6f642..2d0c105e46 100644
--- a/test/framework/module_generator.py
+++ b/test/framework/module_generator.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -266,44 +266,57 @@ def test_load(self):
"""Test load part in generated module file."""
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
- # default: guarded module load (which implies no recursive unloading)
- expected = '\n'.join([
- '',
- "if { ![ is-loaded mod_name ] } {",
- " module load mod_name",
- "}",
- '',
- ])
+ if not self.modtool.supports_safe_auto_load:
+ # default: guarded module load (which implies no recursive unloading)
+ expected = '\n'.join([
+ '',
+ "if { ![ is-loaded mod_name ] } {",
+ " module load mod_name",
+ "}",
+ '',
+ ])
+ else:
+ expected = '\n'.join([
+ '',
+ "module load mod_name",
+ '',
+ ])
self.assertEqual(expected, self.modgen.load_module("mod_name"))
- # with recursive unloading: no if is-loaded guard
- expected = '\n'.join([
- '',
- "if { [ module-info mode remove ] || ![ is-loaded mod_name ] } {",
- " module load mod_name",
- "}",
- '',
- ])
+ if not self.modtool.supports_safe_auto_load:
+ # with recursive unloading: no if is-loaded guard
+ expected = '\n'.join([
+ '',
+ "if { [ module-info mode remove ] || ![ is-loaded mod_name ] } {",
+ " module load mod_name",
+ "}",
+ '',
+ ])
self.assertEqual(expected, self.modgen.load_module("mod_name", recursive_unload=True))
init_config(build_options={'recursive_mod_unload': True})
self.assertEqual(expected, self.modgen.load_module("mod_name"))
# Lmod 7.6+ depends-on
+
+ self.allow_deprecated_behaviour()
+
if self.modtool.supports_depends_on:
expected = '\n'.join([
'',
"depends-on mod_name",
'',
])
- self.assertEqual(expected, self.modgen.load_module("mod_name", depends_on=True))
+ with self.mocked_stdout_stderr():
+ txt = self.modgen.load_module("mod_name", depends_on=True)
+ self.assertEqual(expected, txt)
init_config(build_options={'mod_depends_on': 'True'})
self.assertEqual(expected, self.modgen.load_module("mod_name"))
else:
expected = "depends-on statements in generated module are not supported by modules tool"
- self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name", depends_on=True)
- init_config(build_options={'mod_depends_on': 'True'})
- self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name")
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected,
+ self.modgen.load_module, "mod_name", depends_on=True)
else:
# default: guarded module load (which implies no recursive unloading)
expected = '\n'.join([
@@ -330,20 +343,26 @@ def test_load(self):
self.assertEqual(expected, self.modgen.load_module("mod_name"))
# Lmod 7.6+ depends_on
+
+ self.allow_deprecated_behaviour()
+
if self.modtool.supports_depends_on:
expected = '\n'.join([
'',
'depends_on("mod_name")',
'',
])
- self.assertEqual(expected, self.modgen.load_module("mod_name", depends_on=True))
+ with self.mocked_stdout_stderr():
+ txt = self.modgen.load_module("mod_name", depends_on=True)
+
+ self.assertEqual(expected, txt)
init_config(build_options={'mod_depends_on': 'True'})
self.assertEqual(expected, self.modgen.load_module("mod_name"))
else:
expected = "depends_on statements in generated module are not supported by modules tool"
- self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name", depends_on=True)
- init_config(build_options={'mod_depends_on': 'True'})
- self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name")
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected,
+ self.modgen.load_module, "mod_name", depends_on=True)
def test_load_multi_deps(self):
"""Test generated load statement when multi_deps is involved."""
@@ -353,13 +372,24 @@ def test_load_multi_deps(self):
res = self.modgen.load_module('Python/3.7.4', multi_dep_mods=multi_dep_mods)
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
- expected = '\n'.join([
- '',
- "if { ![ is-loaded Python/3.7.4 ] && ![ is-loaded Python/2.7.16 ] } {",
- " module load Python/3.7.4",
- '}',
- '',
- ])
+ if not self.modtool.supports_safe_auto_load:
+ expected = '\n'.join([
+ '',
+ "if { ![ is-loaded Python/3.7.4 ] && ![ is-loaded Python/2.7.16 ] } {",
+ " module load Python/3.7.4",
+ '}',
+ '',
+ ])
+ else:
+ expected = '\n'.join([
+ '',
+ "if { [ module-info mode remove ] || [ is-loaded Python/2.7.16 ] } {",
+ " module load Python",
+ '} else {',
+ " module load Python/3.7.4",
+ '}',
+ '',
+ ])
else: # Lua syntax
expected = '\n'.join([
'',
@@ -371,8 +401,12 @@ def test_load_multi_deps(self):
self.assertEqual(expected, res)
if self.modtool.supports_depends_on:
+
+ self.allow_deprecated_behaviour()
+
# two versions with depends_on
- res = self.modgen.load_module('Python/3.7.4', multi_dep_mods=multi_dep_mods, depends_on=True)
+ with self.mocked_stdout_stderr():
+ res = self.modgen.load_module('Python/3.7.4', multi_dep_mods=multi_dep_mods, depends_on=True)
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
expected = '\n'.join([
@@ -396,19 +430,33 @@ def test_load_multi_deps(self):
])
self.assertEqual(expected, res)
+ self.disallow_deprecated_behaviour()
+
# now test with more than two versions...
multi_dep_mods = ['foo/1.2.3', 'foo/2.3.4', 'foo/3.4.5', 'foo/4.5.6']
res = self.modgen.load_module('foo/1.2.3', multi_dep_mods=multi_dep_mods)
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
- expected = '\n'.join([
- '',
- "if { ![ is-loaded foo/1.2.3 ] && ![ is-loaded foo/2.3.4 ] && " +
- "![ is-loaded foo/3.4.5 ] && ![ is-loaded foo/4.5.6 ] } {",
- " module load foo/1.2.3",
- '}',
- '',
- ])
+ if not self.modtool.supports_safe_auto_load:
+ expected = '\n'.join([
+ '',
+ "if { ![ is-loaded foo/1.2.3 ] && ![ is-loaded foo/2.3.4 ] && " +
+ "![ is-loaded foo/3.4.5 ] && ![ is-loaded foo/4.5.6 ] } {",
+ " module load foo/1.2.3",
+ '}',
+ '',
+ ])
+ else:
+ expected = '\n'.join([
+ '',
+ "if { [ module-info mode remove ] || [ is-loaded foo/2.3.4 ] || [ is-loaded foo/3.4.5 ] " +
+ "|| [ is-loaded foo/4.5.6 ] } {",
+ " module load foo",
+ "} else {",
+ " module load foo/1.2.3",
+ '}',
+ '',
+ ])
else: # Lua syntax
expected = '\n'.join([
'',
@@ -421,8 +469,12 @@ def test_load_multi_deps(self):
self.assertEqual(expected, res)
if self.modtool.supports_depends_on:
+
+ self.allow_deprecated_behaviour()
+
# more than two versions, with depends_on
- res = self.modgen.load_module('foo/1.2.3', multi_dep_mods=multi_dep_mods, depends_on=True)
+ with self.mocked_stdout_stderr():
+ res = self.modgen.load_module('foo/1.2.3', multi_dep_mods=multi_dep_mods, depends_on=True)
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
expected = '\n'.join([
@@ -448,18 +500,23 @@ def test_load_multi_deps(self):
])
self.assertEqual(expected, res)
+ self.disallow_deprecated_behaviour()
+
# what if we only list a single version?
# see https://github.com/easybuilders/easybuild-framework/issues/3080
res = self.modgen.load_module('one/1.0', multi_dep_mods=['one/1.0'])
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
- expected = '\n'.join([
- '',
- "if { ![ is-loaded one/1.0 ] } {",
- " module load one/1.0",
- '}',
- '',
- ])
+ if not self.modtool.supports_safe_auto_load:
+ expected = '\n'.join([
+ '',
+ "if { ![ is-loaded one/1.0 ] } {",
+ " module load one/1.0",
+ '}',
+ '',
+ ])
+ else:
+ expected = '\nmodule load one/1.0\n'
else: # Lua syntax
expected = '\n'.join([
'',
@@ -471,7 +528,11 @@ def test_load_multi_deps(self):
self.assertEqual(expected, res)
if self.modtool.supports_depends_on:
- res = self.modgen.load_module('one/1.0', multi_dep_mods=['one/1.0'], depends_on=True)
+
+ self.allow_deprecated_behaviour()
+
+ with self.mocked_stdout_stderr():
+ res = self.modgen.load_module('one/1.0', multi_dep_mods=['one/1.0'], depends_on=True)
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
expected = '\ndepends-on one/1.0\n'
@@ -650,7 +711,7 @@ def test_swap(self):
# create tiny test Tcl module to make sure that tested modules tools support single-argument swap
# see https://github.com/easybuilders/easybuild-framework/issues/3396;
- # this is known to fail with the ancient Tcl-only implementation of environment modules,
+ # this is known to fail with the ancient Tcl-only implementation of Environment Modules,
# but that's considered to be a non-issue (since this is mostly relevant for Cray systems,
# which are either using EnvironmentModulesC (3.2.10), EnvironmentModules (4.x) or Lmod...
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl and self.modtool.__class__ != EnvironmentModulesTcl:
@@ -721,13 +782,16 @@ def append_paths(*args, **kwargs):
# check for warning that is printed when same path is added multiple times
with self.modgen.start_module_creation():
self.modgen.append_paths('TEST', 'path1')
- self.mock_stderr(True)
- self.modgen.append_paths('TEST', 'path1')
- stderr = self.get_stderr()
- self.mock_stderr(False)
+ with self.mocked_stdout_stderr():
+ self.modgen.append_paths('TEST', 'path1')
+ stderr = self.get_stderr()
expected_warning = "\nWARNING: Suppressed adding the following path(s) to $TEST of the module "
expected_warning += "as they were already added: path1\n\n"
self.assertEqual(stderr, expected_warning)
+ with self.mocked_stdout_stderr():
+ self.modgen.append_paths('TEST', 'path1', warn_exists=False)
+ stderr = self.get_stderr()
+ self.assertEqual(stderr, '')
def test_module_extensions(self):
"""test the extensions() for extensions"""
@@ -821,14 +885,18 @@ def prepend_paths(*args, **kwargs):
# check for warning that is printed when same path is added multiple times
with self.modgen.start_module_creation():
self.modgen.prepend_paths('TEST', 'path1')
- self.mock_stderr(True)
- self.modgen.prepend_paths('TEST', 'path1')
- stderr = self.get_stderr()
- self.mock_stderr(False)
+ with self.mocked_stdout_stderr():
+ self.modgen.prepend_paths('TEST', 'path1')
+ stderr = self.get_stderr()
expected_warning = "\nWARNING: Suppressed adding the following path(s) to $TEST of the module "
expected_warning += "as they were already added: path1\n\n"
self.assertEqual(stderr, expected_warning)
+ with self.mocked_stdout_stderr():
+ self.modgen.prepend_paths('TEST', 'path1', warn_exists=False)
+ stderr = self.get_stderr()
+ self.assertEqual(stderr, '')
+
def test_det_user_modpath(self):
"""Test for generic det_user_modpath method."""
# None by default
@@ -844,7 +912,10 @@ def test_det_user_modpath(self):
init_config(build_options={'suffix_modules_path': ''})
user_modpath = 'my/{RUNTIME_ENV::TEST123}/modules'
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
- self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my" $::env(TEST123) "modules"')
+ if self.modtool.supports_tcl_getenv:
+ self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my" [getenv TEST123] "modules"')
+ else:
+ self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my" $::env(TEST123) "modules"')
else:
self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my", os.getenv("TEST123"), "modules"')
@@ -900,15 +971,22 @@ def test_getenv_cmd(self):
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
# can't have $LMOD_QUIET set when testing with Tcl syntax,
# otherwise we won't get the output produced by the test module file...
- if 'LMOD_QUIET' in os.environ:
- del os.environ['LMOD_QUIET']
+ os.environ.pop('LMOD_QUIET', None)
- self.assertEqual('$::env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME'))
- self.assertEqual('$::env(HOME)', self.modgen.getenv_cmd('HOME'))
+ if self.modtool.supports_tcl_getenv:
+ self.assertEqual('[getenv HOSTNAME]', self.modgen.getenv_cmd('HOSTNAME'))
+ self.assertEqual('[getenv HOME]', self.modgen.getenv_cmd('HOME'))
- expected = '[if { [info exists ::env(TEST)] } { concat $::env(TEST) } else { concat "foobar" } ]'
- getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar')
- self.assertEqual(getenv_txt, expected)
+ expected = '[getenv TEST "foobar"]'
+ getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar')
+ self.assertEqual(getenv_txt, expected)
+ else:
+ self.assertEqual('$::env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME'))
+ self.assertEqual('$::env(HOME)', self.modgen.getenv_cmd('HOME'))
+
+ expected = '[if { [info exists ::env(TEST)] } { concat $::env(TEST) } else { concat "foobar" } ]'
+ getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar')
+ self.assertEqual(getenv_txt, expected)
write_file(test_mod_file, '#%%Module\nputs stderr %s' % getenv_txt)
else:
@@ -1598,6 +1676,28 @@ def test_generated_module_file_swap(self):
# one/1.0 module was swapped for one/1.1
self.assertEqual(loaded_mods[-2]['mod_name'], 'one/1.1')
+ def test_check_group(self):
+ """Test check_group method."""
+ if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
+ if self.modtool.supports_tcl_check_group:
+ expected = '\n'.join([
+ "if { ![ module-info usergroups group_name ] } {",
+ " error \"mesg\"",
+ "}",
+ '',
+ ])
+ self.assertEqual(expected, self.modgen.check_group("group_name", error_msg="mesg"))
+ else:
+ self.assertEqual('', self.modgen.check_group("group_name", error_msg="mesg"))
+ else:
+ expected = '\n'.join([
+ 'if not ( userInGroup("group_name") ) then',
+ ' LmodError("mesg")',
+ 'end',
+ '',
+ ])
+ self.assertEqual(expected, self.modgen.check_group("group_name", error_msg="mesg"))
+
class TclModuleGeneratorTest(ModuleGeneratorTest):
"""Test for module_generator module for Tcl syntax."""
diff --git a/test/framework/modules.py b/test/framework/modules.py
index a849148bdf..5cd783694d 100644
--- a/test/framework/modules.py
+++ b/test/framework/modules.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -42,7 +42,7 @@
import easybuild.tools.modules as mod
from easybuild.framework.easyblock import EasyBlock
from easybuild.framework.easyconfig.easyconfig import EasyConfig
-from easybuild.tools import StrictVersion
+from easybuild.tools import LooseVersion
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.environment import modify_env
from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir
@@ -51,10 +51,11 @@
from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version
from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches
from easybuild.tools.run import run_shell_cmd
+from easybuild.tools.systemtools import get_shared_lib_ext
# number of modules included for testing purposes
-TEST_MODULES_COUNT = 110
+TEST_MODULES_COUNT = 111
class ModulesTest(EnhancedTestCase):
@@ -99,8 +100,7 @@ def test_run_module(self):
testdir = os.path.dirname(os.path.abspath(__file__))
for key in ['EBROOTGCC', 'EBROOTOPENMPI', 'EBROOTOPENBLAS']:
- if key in os.environ:
- del os.environ[key]
+ os.environ.pop(key, None)
# arguments can be passed in two ways: multiple arguments, or just 1 list argument
self.modtool.run_module('load', 'GCC/6.4.0-2.28')
@@ -122,10 +122,10 @@ def test_run_module(self):
error_pattern = "Module command '.*thisdoesnotmakesense' failed with exit code [1-9]"
self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.run_module, 'thisdoesnotmakesense')
- # we need to use a different error pattern here with EnvironmentModulesC,
+ # we need to use a different error pattern here with Environment Modules,
# because a load of a non-existing module doesnt' trigger a non-zero exit code...
# it will still fail though, just differently
- if isinstance(self.modtool, EnvironmentModulesC):
+ if isinstance(self.modtool, EnvironmentModulesC) or isinstance(self.modtool, EnvironmentModules):
error_pattern = "Unable to locate a modulefile for 'nosuchmodule/1.2.3'"
else:
error_pattern = "Module command '.*load nosuchmodule/1.2.3' failed with exit code [1-9]"
@@ -213,10 +213,8 @@ def test_avail(self):
# test modules include 3 GCC modules and one GCCcore module
ms = self.modtool.available('GCC')
expected = ['GCC/12.3.0', 'GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30']
- # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C do prefix matching
- # EnvironmentModules is a subclass of EnvironmentModulesTcl, but Modules 4+ behaves similarly to Tcl/C impl.,
- # so also append GCCcore/6.2.0 if we are an instance of EnvironmentModules
- if not isinstance(self.modtool, EnvironmentModulesTcl) or isinstance(self.modtool, EnvironmentModules):
+ # ancient Tcl-only Environment Modules tool does an exact match on module name, others do prefix matching
+ if not isinstance(self.modtool, EnvironmentModulesTcl):
expected.extend(['GCCcore/12.3.0', 'GCCcore/6.2.0'])
self.assertEqual(ms, expected)
@@ -226,15 +224,16 @@ def test_avail(self):
# all test modules are accounted for
ms = self.modtool.available()
+ version = LooseVersion(self.modtool.version)
- if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('5.7.5'):
+ if isinstance(self.modtool, Lmod) and version >= '5.7.5' and not version.is_prerelease('5.7.5', ['rc']):
# with recent versions of Lmod, also the hidden modules are included in the output of 'avail'
self.assertEqual(len(ms), TEST_MODULES_COUNT + 3)
self.assertIn('bzip2/.1.0.6', ms)
self.assertIn('toy/.0.0-deps', ms)
self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms)
elif (isinstance(self.modtool, EnvironmentModules)
- and StrictVersion(self.modtool.version) >= StrictVersion('4.6.0')):
+ and version >= '4.6.0' and not version.is_prerelease('4.6.0', ['-beta'])):
# bzip2/.1.0.6 is not there, since that's a module file in Lua syntax
self.assertEqual(len(ms), TEST_MODULES_COUNT + 2)
self.assertIn('toy/.0.0-deps', ms)
@@ -314,7 +313,8 @@ def test_exist(self):
avail_mods = self.modtool.available()
self.assertIn('Java/1.8.0_181', avail_mods)
- if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'):
+ version = LooseVersion(self.modtool.version)
+ if isinstance(self.modtool, Lmod) and version >= '7.0' and not version.is_prerelease('7.0', ['rc']):
self.assertIn('Java/1.8', avail_mods)
self.assertIn('Java/site_default', avail_mods)
self.assertIn('JavaAlias', avail_mods)
@@ -341,12 +341,12 @@ def test_exist(self):
easybuild.tools.modules.MODULE_SHOW_CACHE.clear()
self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True])
- # mimic more verbose stderr output produced by old Tmod version,
- # including a warning produced when multiple .modulerc files are being picked up
+ # mimic "module-*" output produced by EnvironmentModulesC or EnvironmentModulesTcl
+ # mimic warning produced by Environment Modules when a symbol is defined multiple times
# see https://github.com/easybuilders/easybuild-framework/issues/3376
ml_show_java18_stderr = '\n'.join([
"module-version Java/1.8.0_181 1.8",
- "WARNING: Duplicate version symbol '1.8' found",
+ "WARNING: Symbolic version 'Java/1.8' already defined",
"module-version Java/1.8.0_181 1.8",
"-------------------------------------------------------------------",
"/modulefiles/lang/Java/1.8.0_181:",
@@ -374,7 +374,7 @@ def test_exist(self):
self.assertEqual(self.modtool.exist(['Core/Java/1.8', 'Core/Java/site_default']), [True, True])
# also check with .modulerc.lua for Lmod 7.8 or newer
- if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.8'):
+ if isinstance(self.modtool, Lmod) and version >= '7.8' and not version.is_prerelease('7.8', ['rc']):
shutil.move(os.path.join(self.test_prefix, 'Core', 'Java'), java_mod_dir)
reset_module_caches()
@@ -406,7 +406,7 @@ def test_exist(self):
self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True])
# Test alias in home directory .modulerc
- if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'):
+ if isinstance(self.modtool, Lmod) and version >= '7.0' and not version.is_prerelease('7.0', ['rc']):
# Required or temporary HOME would be in MODULEPATH already
self.init_testmods()
# Sanity check: Module aliases don't exist yet
@@ -458,7 +458,7 @@ def test_load(self):
# if GCC is loaded again, $EBROOTGCC should be set again, and GCC should be listed last
self.modtool.load(['GCC/6.4.0-2.28'])
- # environment modules v4+ does not reload already loaded modules
+ # Environment Modules v4+ does not reload already loaded modules
if not isinstance(self.modtool, EnvironmentModules):
self.assertTrue(os.environ.get('EBROOTGCC'))
@@ -467,8 +467,7 @@ def test_load(self):
self.assertEqual(self.modtool.loaded_modules()[-1], 'GCC/6.4.0-2.28')
# set things up for checking that GCC does *not* get reloaded when requested
- if 'EBROOTGCC' in os.environ:
- del os.environ['EBROOTGCC']
+ os.environ.pop('EBROOTGCC', None)
self.modtool.load(['OpenMPI/2.1.2-GCC-6.4.0-2.28'])
if isinstance(self.modtool, Lmod):
# order of loaded modules only changes with Lmod
@@ -691,10 +690,23 @@ def test_get_software_root_version_libdir(self):
os.environ.pop('EBROOT%s' % env_var_name)
os.environ.pop('EBVERSION%s' % env_var_name)
- # check expected result of get_software_libdir with multiple lib subdirs
+ # if only 'lib' has a library archive, use it
root = os.path.join(tmpdir, name)
mkdir(os.path.join(root, 'lib64'))
os.environ['EBROOT%s' % env_var_name] = root
+ write_file(os.path.join(root, 'lib', 'libfoo.a'), 'foo')
+ self.assertEqual(get_software_libdir(name), 'lib')
+
+ remove_file(os.path.join(root, 'lib', 'libfoo.a'))
+
+ # also check vice versa with *shared* library in lib64
+ shlib_ext = get_shared_lib_ext()
+ write_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext), 'foo')
+ self.assertEqual(get_software_libdir(name), 'lib64')
+
+ remove_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext))
+
+ # check expected result of get_software_libdir with multiple lib subdirs
self.assertErrorRegex(EasyBuildError, "Multiple library subdirectories found.*", get_software_libdir, name)
self.assertEqual(get_software_libdir(name, only_one=False), ['lib', 'lib64'])
@@ -1045,8 +1057,7 @@ def test_modules_tool_stateless(self):
init_config()
# make sure $LMOD_DEFAULT_MODULEPATH, since Lmod picks it up and tweaks $MODULEPATH to match it
- if 'LMOD_DEFAULT_MODULEPATH' in os.environ:
- del os.environ['LMOD_DEFAULT_MODULEPATH']
+ os.environ.pop('LMOD_DEFAULT_MODULEPATH', None)
self.reset_modulepath([os.path.join(self.test_prefix, 'Core')])
@@ -1068,8 +1079,7 @@ def test_modules_tool_stateless(self):
self.modtool.load(['OpenMPI/2.1.2'])
self.modtool.purge()
- if 'LMOD_DEFAULT_MODULEPATH' in os.environ:
- del os.environ['LMOD_DEFAULT_MODULEPATH']
+ os.environ.pop('LMOD_DEFAULT_MODULEPATH', None)
# reset $MODULEPATH, obtain new ModulesTool instance,
# which should not remember anything w.r.t. previous $MODULEPATH value
@@ -1401,7 +1411,7 @@ def test_exit_code_check(self):
if isinstance(self.modtool, Lmod):
error_pattern = "Module command '.*load nosuchmoduleavailableanywhere' failed with exit code"
else:
- # Tcl implementations exit with 0 even when a non-existing module is loaded...
+ # Environment Modules exits with 0 even when a non-existing module is loaded...
error_pattern = "Unable to locate a modulefile for 'nosuchmoduleavailableanywhere'"
self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.load, ['nosuchmoduleavailableanywhere'])
diff --git a/test/framework/modules/intel-compilers/2024.0.0 b/test/framework/modules/intel-compilers/2024.0.0
new file mode 100644
index 0000000000..a5c0267f9d
--- /dev/null
+++ b/test/framework/modules/intel-compilers/2024.0.0
@@ -0,0 +1,37 @@
+#%Module
+proc ModulesHelp { } {
+ puts stderr {
+
+Description
+===========
+Intel C, C++ & Fortran compilers (classic and oneAPI)
+
+
+More information
+================
+ - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html
+ }
+}
+
+module-whatis {Description: Intel C, C++ & Fortran compilers (classic and oneAPI)}
+module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html}
+module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html}
+
+set root /tmp/intel-compilers/2024.0.0
+
+conflict intel-compilers
+
+prepend-path CPATH $root/tbb/2021.11/include
+prepend-path LD_LIBRARY_PATH $root/compiler/2024.0/linux/lib
+prepend-path LD_LIBRARY_PATH $root/tbb/2021.11/lib/intel64/gcc4.8
+prepend-path LIBRARY_PATH $root/compiler/2024.0/linux/lib
+prepend-path LIBRARY_PATH $root/tbb/2021.11/lib/intel64/gcc4.8
+prepend-path MANPATH $root/compiler/2024.0/share/man
+prepend-path OCL_ICD_FILENAMES $root/compiler/2024.0/lib/libintelocl.so
+prepend-path PATH $root/compiler/2024.0/bin
+prepend-path TBBROOT $root/tbb/2021.11
+setenv EBROOTINTELMINCOMPILERS "$root"
+setenv EBVERSIONINTELMINCOMPILERS "2024.0.0"
+setenv EBDEVELINTELMINCOMPILERS "$root/easybuild/Core-intel-compilers-2024.0.0-easybuild-devel"
+
+# Built with EasyBuild version 4.8.2
diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py
index f43a91e3b3..3caa1772ce 100644
--- a/test/framework/modulestool.py
+++ b/test/framework/modulestool.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -36,7 +36,7 @@
from unittest import TextTestRunner
from easybuild.base import fancylogger
-from easybuild.tools import modules, StrictVersion
+from easybuild.tools import modules, LooseVersion
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import read_file, which, write_file
from easybuild.tools.modules import EnvironmentModules, Lmod
@@ -76,7 +76,7 @@ def test_mock(self):
mmt = MockModulesTool(mod_paths=[], testing=True)
# the version of the MMT is the commandline option
- self.assertEqual(mmt.version, StrictVersion(MockModulesTool.VERSION_OPTION))
+ self.assertEqual(mmt.version, LooseVersion(MockModulesTool.VERSION_OPTION))
cmd_abspath = which(MockModulesTool.COMMAND)
@@ -100,7 +100,7 @@ def test_environment_command(self):
bmmt = BrokenMockModulesTool(mod_paths=[], testing=True)
cmd_abspath = which(MockModulesTool.COMMAND)
- self.assertEqual(bmmt.version, StrictVersion(MockModulesTool.VERSION_OPTION))
+ self.assertEqual(bmmt.version, LooseVersion(MockModulesTool.VERSION_OPTION))
self.assertEqual(bmmt.cmd, cmd_abspath)
# clean it up
@@ -209,12 +209,36 @@ def test_environment_modules_specific(self):
mt = EnvironmentModules(testing=True)
self.assertIsInstance(mt.loaded_modules(), list) # dummy usage
+ # test updating module cache
+ test_modulepath = os.path.join(self.test_installpath, 'modules', 'all')
+ os.environ['MODULEPATH'] = test_modulepath
+ test_module_dir = os.path.join(test_modulepath, 'test')
+ test_module_file = os.path.join(test_module_dir, '1.2.3')
+ write_file(test_module_file, '#%Module')
+ build_options = {
+ 'update_modules_tool_cache': True,
+ }
+ init_config(build_options=build_options)
+ mt = EnvironmentModules(testing=True)
+ out = mt.update()
+ os.remove(test_module_file)
+ os.rmdir(test_module_dir)
+
+ # test cache file has been created if module tool supports it
+ if LooseVersion(mt.version) >= LooseVersion('5.3.0'):
+ cache_fp = os.path.join(test_modulepath, '.modulecache')
+ expected = "Creating %s\n" % cache_fp
+ self.assertEqual(expected, out, "Module cache created")
+ self.assertTrue(os.path.exists(cache_fp))
+ os.remove(cache_fp)
+
# initialize Environment Modules tool with non-official version number
# pass (fake) full path to 'modulecmd.tcl' via $MODULES_CMD
fake_path = os.path.join(self.test_installpath, 'libexec', 'modulecmd.tcl')
fake_modulecmd_txt = '\n'.join([
- 'puts stderr {Modules Release 5.3.1+unload-188-g14b6b59b (2023-10-21)}',
- "puts {os.environ['FOO'] = 'foo'}",
+ '#!/bin/bash',
+ 'echo "Modules Release 5.3.1+unload-188-g14b6b59b (2023-10-21)" >&2',
+ 'echo "os.environ[\'FOO\'] = \'foo\'"',
])
write_file(fake_path, fake_modulecmd_txt)
os.chmod(fake_path, stat.S_IRUSR | stat.S_IXUSR)
@@ -232,8 +256,7 @@ def tearDown(self):
if self.orig_module is not None:
os.environ['module'] = self.orig_module
else:
- if 'module' in os.environ:
- del os.environ['module']
+ os.environ.pop('module', None)
def suite():
diff --git a/test/framework/options.py b/test/framework/options.py
index 92aeb7fbf9..7ffe5f9ffd 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -65,17 +65,14 @@
from easybuild.tools.options import set_up_configuration, set_tmpdir, use_color
from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX
from easybuild.tools.run import run_shell_cmd
-from easybuild.tools.systemtools import HAVE_ARCHSPEC
+from easybuild.tools.systemtools import DARWIN, HAVE_ARCHSPEC, get_os_type
from easybuild.tools.version import VERSION
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup, init_config
try:
import pycodestyle # noqa
except ImportError:
- try:
- import pep8 # noqa
- except ImportError:
- pass
+ pass
EXTERNAL_MODULES_METADATA = """[foobar/1.2.3]
@@ -377,6 +374,26 @@ def test_skip(self):
self.assertEqual(len(glob.glob(toy_mod_glob)), 1)
+ # check use of module_only parameter
+ remove_dir(os.path.join(self.test_installpath, 'modules', 'all', 'toy'))
+ remove_dir(os.path.join(self.test_installpath, 'software', 'toy', '0.0'))
+ args = [
+ test_ec,
+ '--rebuild',
+ ]
+ test_ec_txt += "\nmodule_only = True\n"
+ write_file(test_ec, test_ec_txt)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True, raise_error=True)
+
+ self.assertEqual(len(glob.glob(toy_mod_glob)), 1)
+
+ # check that no software was installed
+ installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
+ installdir_glob = glob.glob(os.path.join(installdir, '*'))
+ easybuild_dir = os.path.join(installdir, 'easybuild')
+ self.assertEqual(installdir_glob, [easybuild_dir])
+
def test_skip_test_step(self):
"""Test skipping testing the build (--skip-test-step)."""
@@ -435,6 +452,30 @@ def test_ignore_test_failure(self):
error_pattern = 'Found both ignore-test-failure and skip-test-step enabled'
self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+ def test_skip_sanity_check(self):
+ """Test skipping of sanity check step (--skip-sanity-check)."""
+
+ topdir = os.path.abspath(os.path.dirname(__file__))
+ toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, read_file(toy_ec) + "\nsanity_check_commands = ['this_will_fail']")
+
+ args = [test_ec, '--rebuild']
+ err_msg = "Sanity check failed"
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, args, do_build=True, raise_error=True)
+
+ args.append('--skip-sanity-check')
+ with self.mocked_stdout_stderr():
+ outtext = self.eb_main(args, do_build=True, raise_error=True)
+ self.assertNotIn('sanity checking...', outtext)
+
+ # Passing skip and only options is disallowed
+ args.append('--sanity-check-only')
+ error_pattern = 'Found both skip-sanity-check and sanity-check-only enabled'
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+
def test_job(self):
"""Test submitting build as a job."""
@@ -584,6 +625,8 @@ def run_test(fmt=None):
r'^``%\(arch\)s``\s+System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)\s*$',
r'^``%\(cuda_cc_space_sep\)s``\s+Space-separated list of CUDA compute capabilities\s*$',
r'^``SOURCE_TAR_GZ``\s+Source \.tar\.gz bundle\s+``%\(name\)s-%\(version\)s.tar.gz``\s*$',
+ r'^``%\(software_commit\)s``\s+Git commit id to use for the software as specified '
+ 'by --software-commit command line option',
]
else:
pattern_lines = [
@@ -596,6 +639,8 @@ def run_test(fmt=None):
r'^\s+%\(arch\)s: System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)$',
r'^\s+%\(cuda_cc_space_sep\)s: Space-separated list of CUDA compute capabilities$',
r'^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)$',
+ r'^\s+%\(software_commit\)s: Git commit id to use for the software as specified '
+ 'by --software-commit command line option',
]
for pattern_line in pattern_lines:
@@ -840,7 +885,7 @@ def test_avail_lists(self):
os.close(fd)
name_items = {
- 'modules-tools': ['EnvironmentModulesC', 'Lmod'],
+ 'modules-tools': ['EnvironmentModules', 'Lmod'],
'module-naming-schemes': ['EasyBuildMNS', 'HierarchicalMNS', 'CategorizedHMNS'],
}
for (name, items) in name_items.items():
@@ -856,7 +901,7 @@ def test_avail_lists(self):
info_msg = r"INFO List of supported %s:" % words
self.assertTrue(re.search(info_msg, logtxt), "Info message with list of available %s" % words)
for item in items:
- res = re.findall(r"^\s*%s" % item, logtxt, re.M)
+ res = re.findall(r"^\s*%s\n" % item, logtxt, re.M)
self.assertTrue(res, "%s is included in list of available %s" % (item, words))
# every item should only be mentioned once
n = len(res)
@@ -923,37 +968,46 @@ def test_000_list_easyblocks(self):
logtxt = read_file(self.logfile)
expected = '\n'.join([
- r'EasyBlock',
- r'\|-- bar',
- r'\|-- ConfigureMake',
- r'\| \|-- MakeCp',
- r'\|-- EB_EasyBuildMeta',
- r'\|-- EB_FFTW',
- r'\|-- EB_foo',
- r'\| \|-- EB_foofoo',
- r'\|-- EB_GCC',
- r'\|-- EB_HPL',
- r'\|-- EB_libtoy',
- r'\|-- EB_OpenBLAS',
- r'\|-- EB_OpenMPI',
- r'\|-- EB_ScaLAPACK',
- r'\|-- EB_toy_buggy',
- r'\|-- ExtensionEasyBlock',
- r'\| \|-- DummyExtension',
- r'\| \|-- EB_toy',
- r'\| \| \|-- EB_toy_eula',
- r'\| \| \|-- EB_toytoy',
- r'\| \|-- Toy_Extension',
- r'\|-- ModuleRC',
- r'\|-- PythonBundle',
- r'\|-- Toolchain',
- r'Extension',
- r'\|-- ExtensionEasyBlock',
- r'\| \|-- DummyExtension',
- r'\| \|-- EB_toy',
- r'\| \| \|-- EB_toy_eula',
- r'\| \| \|-- EB_toytoy',
- r'\| \|-- Toy_Extension',
+ "EasyBlock",
+ "|-- bar",
+ "|-- ConfigureMake",
+ "| |-- MakeCp",
+ "|-- EB_EasyBuildMeta",
+ "|-- EB_FFTW",
+ "|-- EB_foo",
+ "| |-- EB_foofoo",
+ "|-- EB_GCC",
+ "|-- EB_HPL",
+ "|-- EB_libtoy",
+ "|-- EB_OpenBLAS",
+ "|-- EB_OpenMPI",
+ "|-- EB_ScaLAPACK",
+ "|-- EB_toy_buggy",
+ "|-- ExtensionEasyBlock",
+ "| |-- DummyExtension",
+ "| | |-- CustomDummyExtension",
+ "| | | |-- ChildCustomDummyExtension",
+ "| | |-- DeprecatedDummyExtension",
+ "| | | |-- ChildDeprecatedDummyExtension",
+ "| |-- EB_toy",
+ "| | |-- EB_toy_eula",
+ "| | |-- EB_toytoy",
+ "| |-- Toy_Extension",
+ "|-- ModuleRC",
+ "|-- PythonBundle",
+ "|-- Toolchain",
+ "Extension",
+ "|-- ExtensionEasyBlock",
+ "| |-- DummyExtension",
+ "| | |-- CustomDummyExtension",
+ "| | | |-- ChildCustomDummyExtension",
+ "| | |-- DeprecatedDummyExtension",
+ "| | | |-- ChildDeprecatedDummyExtension",
+ "| |-- EB_toy",
+ "| | |-- EB_toy_eula",
+ "| | |-- EB_toytoy",
+ "| |-- Toy_Extension",
+ "",
])
regex = re.compile(expected, re.M)
self.assertTrue(regex.search(logtxt), "Pattern '%s' found in: %s" % (regex.pattern, logtxt))
@@ -1444,14 +1498,81 @@ def test_github_copy_ec_from_pr(self):
self.assertIn("name = 'ExifTool'", read_file(test_ec))
remove_file(test_ec)
+ def test_copy_ec_from_commit(self):
+ """Test combination of --copy-ec with --from-commit."""
+ # note: --from-commit does not involve using GitHub API, so no GitHub token required
+
+ # using easyconfigs commit to add EasyBuild-4.8.2.eb
+ test_commit = '7c83a553950c233943c7b0189762f8c05cfea852'
+
+ test_dir = os.path.join(self.test_prefix, 'from_commit')
+ mkdir(test_dir, parents=True)
+ args = ['--copy-ec', '--from-commit=%s' % test_commit, test_dir]
+ try:
+ stdout = self.mocked_main(args)
+ except URLError as err:
+ print("Ignoring URLError '%s' in test_copy_ec_from_commit" % err)
+
+ pattern = "_%s/e/EasyBuild/EasyBuild-4.8.2.eb copied to " % test_commit
+ self.assertIn(pattern, stdout)
+ copied_ecs = os.listdir(test_dir)
+ self.assertEqual(copied_ecs, ['EasyBuild-4.8.2.eb'])
+
+ # cleanup
+ remove_dir(test_dir)
+ mkdir(test_dir)
+
+ # test again, using extra argument (name of file to copy), without specifying target directory
+ # (should copy to current directory)
+ cwd = change_dir(test_dir)
+ args = ['--copy-ec', '--from-commit=%s' % test_commit, "EasyBuild-4.8.2.eb"]
+ try:
+ stdout = self.mocked_main(args)
+ except URLError as err:
+ print("Ignoring URLError '%s' in test_copy_ec_from_commit" % err)
+
+ self.assertIn(pattern, stdout)
+ copied_ecs = os.listdir(test_dir)
+ self.assertEqual(copied_ecs, ['EasyBuild-4.8.2.eb'])
+
+ # cleanup
+ change_dir(cwd)
+ remove_dir(test_dir)
+ mkdir(test_dir)
+
+ # test with commit that touches a bunch of easyconfigs
+ test_commit = '49c887397b1a948e1909fc24bc905fdc1ad38388'
+ expected_ecs = [
+ 'gompi-2023b.eb',
+ 'gfbf-2023b.eb',
+ 'ScaLAPACK-2.2.0-gompi-2023b-fb.eb',
+ 'foss-2023b.eb',
+ 'HPL-2.3-foss-2023b.eb',
+ 'FFTW.MPI-3.3.10-gompi-2023b.eb',
+ 'SciPy-bundle-2023.11-gfbf-2023b.eb',
+ 'OSU-Micro-Benchmarks-7.2-gompi-2023b.eb',
+ ]
+ args = ['--copy-ec', '--from-commit=%s' % test_commit, test_dir]
+ try:
+ stdout = self.mocked_main(args)
+ except URLError as err:
+ print("Ignoring URLError '%s' in test_copy_ec_from_commit" % err)
+
+ copied_ecs = os.listdir(test_dir)
+ for ec in expected_ecs:
+ self.assertIn(ec, copied_ecs)
+
def test_dry_run(self):
"""Test dry run (long format)."""
+
+ # first test with --robot
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
os.close(fd)
args = [
'gzip-1.4-GCC-4.6.3.eb',
- '--dry-run', # implies enabling dependency resolution
+ '--dry-run',
+ '--robot', # implies enabling dependency resolution
'--unittest-file=%s' % self.logfile,
]
with self.mocked_stdout_stderr():
@@ -1468,6 +1589,24 @@ def test_dry_run(self):
regex = re.compile(r" \* \[%s\] \S+%s \(module: %s\)" % (mark, ec, mod), re.M)
self.assertTrue(regex.search(logtxt), "Found match for pattern %s in '%s'" % (regex.pattern, logtxt))
+ # next test without --robot
+ fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
+ os.close(fd)
+
+ args = [
+ 'gzip-1.4-GCC-4.6.3.eb',
+ '--dry-run',
+ '--unittest-file=%s' % self.logfile,
+ ]
+ self.eb_main(args, logfile=dummylogfn)
+ logtxt = read_file(self.logfile)
+
+ info_msg = r"Dry run: printing build status of easyconfigs"
+ self.assertTrue(re.search(info_msg, logtxt, re.M), "Info message dry running in '%s'" % logtxt)
+ ec, mod, mark = ("gzip-1.4-GCC-4.6.3.eb", "gzip/1.4-GCC-4.6.3", ' ')
+ regex = re.compile(r" \* \[%s\] \S+%s \(module: %s\)" % (mark, ec, mod), re.M)
+ self.assertTrue(regex.search(logtxt), "Found match for pattern %s in '%s'" % (regex.pattern, logtxt))
+
def test_missing(self):
"""Test use of --missing/-M."""
@@ -1495,14 +1634,26 @@ def test_missing(self):
])
for opt in ['-M', '--missing-modules']:
- self.mock_stderr(True)
- self.mock_stdout(True)
- self.eb_main(args + [opt], testing=False, raise_error=True)
- stderr, stdout = self.get_stderr(), self.get_stdout()
- self.mock_stderr(False)
- self.mock_stdout(False)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args + [opt], testing=False, raise_error=True)
+ stderr, stdout = self.get_stderr(), self.get_stdout()
self.assertFalse(stderr)
self.assertIn(expected, stdout)
+ # --terse
+ with self.mocked_stdout_stderr():
+ self.eb_main(args + ['-M', '--terse'], testing=False, raise_error=True)
+ stderr, stdout = self.get_stderr(), self.get_stdout()
+ self.assertFalse(stderr)
+ if mns == 'HierarchicalMNS':
+ expected = '\n'.join([
+ "GCC-4.6.3.eb",
+ "intel-2018a.eb",
+ "toy-0.0-deps.eb",
+ "gzip-1.4-GCC-4.6.3.eb",
+ ])
+ else:
+ expected = 'gzip-1.4-GCC-4.6.3.eb'
+ self.assertEqual(stdout, expected + '\n')
def test_dry_run_short(self):
"""Test dry run (short format)."""
@@ -1610,6 +1761,7 @@ def test_try_toolchain_mapping(self):
gzip_ec,
'--try-toolchain=iccifort,2016.1.150-GCC-4.9.3-2.25',
'--dry-run',
+ '--robot',
]
# by default, toolchain mapping is enabled
@@ -1666,6 +1818,7 @@ def test_try_update_deps(self):
'--try-toolchain-version=6.4.0-2.28',
'--try-update-deps',
'-D',
+ '--robot',
]
with self.mocked_stdout_stderr():
@@ -1750,6 +1903,7 @@ def test_dry_run_hierarchical(self):
'gzip-1.5-foss-2018a.eb',
'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb',
'--dry-run',
+ '--robot',
'--unittest-file=%s' % self.logfile,
'--module-naming-scheme=HierarchicalMNS',
'--ignore-osdeps',
@@ -1791,6 +1945,7 @@ def test_dry_run_categorized(self):
'gzip-1.5-foss-2018a.eb',
'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb',
'--dry-run',
+ '--robot',
'--unittest-file=%s' % self.logfile,
'--module-naming-scheme=CategorizedHMNS',
'--ignore-osdeps',
@@ -1865,7 +2020,7 @@ def test_github_from_pr(self):
# make sure that *only* these modules are listed, no others
regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M)
- self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules))
+ self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules))
pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr6424')
regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M)
@@ -1901,12 +2056,12 @@ def test_github_from_pr(self):
# make sure that *only* these modules are listed, no others
regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M)
- self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules))
+ self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules))
for pr in ('12150', '12366'):
pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr%s' % pr)
regex = re.compile(r"Extended list of robot search paths with .*%s.*:" % pr_tmpdir, re.M)
- self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
+ self.assertTrue(regex.search(outtxt), "Found pattern '%s' in: %s" % (regex.pattern, outtxt))
except URLError as err:
print("Ignoring URLError '%s' in test_from_pr" % err)
@@ -2046,6 +2201,137 @@ def test_github_from_pr_x(self):
except URLError as err:
print("Ignoring URLError '%s' in test_from_pr_x" % err)
+ def test_from_commit(self):
+ """Test for --from-commit."""
+ # note: --from-commit does not involve using GitHub API, so no GitHub token required
+
+ # easyconfigs commit to add EasyBuild-4.8.2.eb
+ test_commit = '7c83a553950c233943c7b0189762f8c05cfea852'
+
+ fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
+ os.close(fd)
+
+ tmpdir = tempfile.mkdtemp()
+ args = [
+ '--from-commit=%s' % test_commit,
+ '--dry-run',
+ '--tmpdir=%s' % tmpdir,
+ ]
+ try:
+ outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ modules = [
+ (tmpdir, 'EasyBuild/4.8.2'),
+ ]
+ for path_prefix, module in modules:
+ ec_fn = "%s.eb" % '-'.join(module.split('/'))
+ path = '.*%s' % os.path.dirname(path_prefix)
+ regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path, ec_fn, module), re.M)
+ self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
+
+ # make sure that *only* these modules are listed, no others
+ regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M)
+ self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules))
+
+ pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_commit_%s' % test_commit)
+ regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M)
+ self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
+ except URLError as err:
+ print("Ignoring URLError '%s' in test_from_commit" % err)
+ shutil.rmtree(tmpdir)
+
+ easyblock_template = '\n'.join([
+ "from easybuild.framework.easyblock import EasyBlock",
+ "class %s(EasyBlock):",
+ " pass",
+ ])
+
+ # create fake custom easyblock for CMake that is required by easyconfig used in test below
+ easyblock_file = os.path.join(self.test_prefix, 'easyblocks', 'cmake.py')
+ write_file(easyblock_file, easyblock_template % 'EB_CMake')
+
+ # also test with an easyconfig that requires additional easyconfigs to resolve dependencies,
+ # cfr. https://github.com/easybuilders/easybuild-framework/issues/4540;
+ # using commit that adds CMake-3.18.4.eb (which requires ncurses-6.2.eb),
+ # see https://github.com/easybuilders/easybuild-easyconfigs/pull/13156
+ test_commit = '41eee3fe2e5102f52319481ca8dde16204dab590'
+ args = [
+ '--from-commit=%s' % test_commit,
+ '--dry-run',
+ '--robot',
+ '--tmpdir=%s' % tmpdir,
+ '--include-easyblocks=' + os.path.join(self.test_prefix, 'easyblocks', '*.py'),
+ ]
+ try:
+ outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ modules = [
+ (tmpdir, 'ncurses/6.2'),
+ (tmpdir, 'CMake/3.18.4'),
+ ]
+ for path_prefix, module in modules:
+ ec_fn = "%s.eb" % '-'.join(module.split('/'))
+ path = '.*%s' % os.path.dirname(path_prefix)
+ regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path, ec_fn, module), re.M)
+ self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
+
+ # make sure that *only* these modules are listed, no others
+ regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M)
+ self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules))
+
+ pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_commit_%s' % test_commit)
+ regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M)
+ self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
+ except URLError as err:
+ print("Ignoring URLError '%s' in test_from_commit" % err)
+ shutil.rmtree(tmpdir)
+
+ # must be run after test for --list-easyblocks, hence the '_xxx_'
+ # cleaning up the imported easyblocks is quite difficult...
+ def test_xxx_include_easyblocks_from_commit(self):
+ """Test for --include-easyblocks-from-commit."""
+ # note: --include-easyblocks-from-commit does not involve using GitHub API, so no GitHub token required
+
+ orig_local_sys_path = sys.path[:]
+ # easyblocks commit only touching Binary easyblock
+ test_commit = '94d28c556947bd96d0978df775b15a50a4600c6f'
+
+ fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
+ os.close(fd)
+
+ tmpdir = tempfile.mkdtemp()
+ args = [
+ '--include-easyblocks-from-commit=%s' % test_commit,
+ '--dry-run',
+ '--tmpdir=%s' % tmpdir,
+ 'toy-0.0.eb', # test easyconfig
+ ]
+ try:
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+
+ # 'undo' import of foo easyblock
+ del sys.modules['easybuild.easyblocks.generic.binary']
+ sys.path[:] = orig_local_sys_path
+ import easybuild.easyblocks
+ reload(easybuild.easyblocks)
+ import easybuild.easyblocks.generic
+ reload(easybuild.easyblocks.generic)
+
+ pattern = "== easyblock binary.py included from commit %s" % test_commit
+ self.assertEqual(stderr, '')
+ self.assertIn(pattern, stdout)
+
+ regex = re.compile(r"^ \* \[.\] .*/toy-0.0.eb \(module: toy/0.0\)$", re.M)
+ self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
+
+ except URLError as err:
+ print("Ignoring URLError '%s' in test_include_easyblocks_from_commit" % err)
+ shutil.rmtree(tmpdir)
+
def test_no_such_software(self):
"""Test using no arguments."""
@@ -2838,6 +3124,7 @@ def test_hide_toolchains(self):
args = [
ec_file,
'--dry-run',
+ '--robot',
'--hide-toolchains=GCC',
]
with self.mocked_stdout_stderr():
@@ -3238,8 +3525,7 @@ def test_show_default_configfiles(self):
home = os.environ['HOME']
for envvar in ['XDG_CONFIG_DIRS', 'XDG_CONFIG_HOME']:
- if envvar in os.environ:
- del os.environ[envvar]
+ os.environ.pop(envvar, None)
reload(easybuild.tools.options)
args = [
@@ -3259,7 +3545,7 @@ def test_show_default_configfiles(self):
'',
"* user-level: ${XDG_CONFIG_HOME:-$HOME/.config}/easybuild/config.cfg",
" -> %s",
- "* system-level: ${XDG_CONFIG_DIRS:-/etc}/easybuild.d/*.cfg",
+ "* system-level: ${XDG_CONFIG_DIRS:-/etc/xdg}/easybuild.d/*.cfg",
" -> %s/easybuild.d/*.cfg => ",
])
@@ -3274,18 +3560,19 @@ def test_show_default_configfiles(self):
homecfgfile_str += " => found"
else:
homecfgfile_str += " => not found"
- expected = expected_tmpl % ('(not set)', '(not set)', homecfgfile_str, '{/etc}')
+ expected = expected_tmpl % ('(not set)', '(not set)', homecfgfile_str, '{/etc/xdg}')
self.assertIn(expected, logtxt)
# to predict the full output, we need to take control over $HOME and $XDG_CONFIG_DIRS
os.environ['HOME'] = self.test_prefix
- xdg_config_dirs = os.path.join(self.test_prefix, 'etc')
+ xdg_config_dirs = os.path.join(self.test_prefix, 'etc', 'xdg')
os.environ['XDG_CONFIG_DIRS'] = xdg_config_dirs
expected_tmpl += '\n'.join([
"%s",
'',
- "Default list of existing configuration files (%d): %s",
+ "Default list of existing configuration files (%d, most important last):",
+ "%s",
])
# put dummy cfgfile in place in $HOME (to predict last line of output which only lists *existing* files)
@@ -3304,12 +3591,12 @@ def test_show_default_configfiles(self):
xdg_config_home = os.path.join(self.test_prefix, 'home')
os.environ['XDG_CONFIG_HOME'] = xdg_config_home
- xdg_config_dirs = [os.path.join(self.test_prefix, 'etc'), os.path.join(self.test_prefix, 'moaretc')]
+ xdg_config_dirs = [os.path.join(self.test_prefix, 'moaretc'), os.path.join(self.test_prefix, 'etc', 'xdg')]
os.environ['XDG_CONFIG_DIRS'] = os.pathsep.join(xdg_config_dirs)
# put various dummy cfgfiles in place
cfgfiles = [
- os.path.join(self.test_prefix, 'etc', 'easybuild.d', 'config.cfg'),
+ os.path.join(self.test_prefix, 'etc', 'xdg', 'easybuild.d', 'config.cfg'),
os.path.join(self.test_prefix, 'moaretc', 'easybuild.d', 'bar.cfg'),
os.path.join(self.test_prefix, 'moaretc', 'easybuild.d', 'foo.cfg'),
os.path.join(xdg_config_home, 'easybuild', 'config.cfg'),
@@ -3326,7 +3613,7 @@ def test_show_default_configfiles(self):
expected = expected_tmpl % (xdg_config_home, os.pathsep.join(xdg_config_dirs),
"%s => found" % os.path.join(xdg_config_home, 'easybuild', 'config.cfg'),
'{' + ', '.join(xdg_config_dirs) + '}',
- ', '.join(cfgfiles[:-1]), 4, ', '.join(cfgfiles))
+ ', '.join(cfgfiles[1:3]+[cfgfiles[0]]), 4, ', '.join(cfgfiles))
self.assertIn(expected, logtxt)
del os.environ['XDG_CONFIG_DIRS']
@@ -4102,6 +4389,7 @@ def test_minimal_toolchains(self):
'--minimal-toolchains',
'--module-naming-scheme=HierarchicalMNS',
'--dry-run',
+ '--robot',
]
self.mock_stdout(True)
self.eb_main(args, do_build=True, raise_error=True, testing=False)
@@ -4315,7 +4603,7 @@ def test_new_branch_github(self):
def test_github_new_pr_from_branch(self):
"""Test --new-pr-from-branch."""
if self.github_token is None:
- print("Skipping test_new_pr_from_branch, no GitHub token available?")
+ print("Skipping test_github_new_pr_from_branch, no GitHub token available?")
return
# see https://github.com/boegel/easybuild-easyconfigs/tree/test_new_pr_from_branch_DO_NOT_REMOVE
@@ -4347,8 +4635,8 @@ def test_github_new_pr_from_branch(self):
r"^\* from: boegel/easybuild-easyconfigs:test_new_pr_from_branch_DO_NOT_REMOVE$",
r'^\* title: "\{tools\}\[system/system\] toy v0\.0"$',
r'^"an easyconfig for toy"$',
- r"^ 1 file changed, 32 insertions\(\+\)$",
- r"^\* overview of changes:\n easybuild/easyconfigs/t/toy/toy-0\.0\.eb | 32",
+ r"^ 1 file changed, [0-9]+ insertions\(\+\)$",
+ r"^\* overview of changes:\n easybuild/easyconfigs/t/toy/toy-0\.0\.eb | [0-9]+",
]
self._assert_regexs(regexs, txt)
@@ -4376,7 +4664,7 @@ def test_update_branch_github(self):
r"^== fetching branch 'develop' from https://github.com/%s.git\.\.\." % full_repo,
r"^== copying files to .*/git-working-dir.*/easybuild-easyconfigs...",
r"^== pushing branch 'develop' to remote '.*' \(git@github.com:%s.git\) \[DRY RUN\]" % full_repo,
- r"^Overview of changes:\n.*/easyconfigs/t/toy/toy-0.0.eb \| 32",
+ r"^Overview of changes:\n.*/easyconfigs/t/toy/toy-0.0.eb \| [0-9]+",
r"== pushed updated branch 'develop' to boegel/easybuild-easyconfigs \[DRY RUN\]",
]
self._assert_regexs(regexs, txt)
@@ -4449,7 +4737,7 @@ def test_github_new_update_pr(self):
'--git-working-dirs-path=%s' % git_working_dir,
':bzip2-1.0.6.eb',
])
- error_msg = "A meaningful commit message must be specified via --pr-commit-msg"
+ error_msg = "A meaningful commit message must be specified via --pr-commit-msg.*\nDeleted: bzip2-1.0.6.eb"
self.mock_stdout(True)
self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True, testing=False)
@@ -4460,7 +4748,7 @@ def test_github_new_update_pr(self):
res = [d for d in res if os.path.basename(d) != os.path.basename(git_working_dir)]
if len(res) == 1:
unstaged_file_full = os.path.join(res[0], unstaged_file)
- self.assertNotExists(unstaged_file_full), "%s not found in %s" % (unstaged_file, res[0])
+ self.assertNotExists(unstaged_file_full)
else:
self.fail("Found copy of easybuild-easyconfigs working copy")
@@ -4528,7 +4816,8 @@ def test_github_new_update_pr(self):
gcc_ec,
'-D',
]
- error_msg = "A meaningful commit message must be specified via --pr-commit-msg"
+ error_msg = "A meaningful commit message must be specified via --pr-commit-msg.*\n"
+ error_msg += "Modified: " + os.path.basename(gcc_ec)
self.mock_stdout(True)
self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True)
self.mock_stdout(False)
@@ -5024,6 +5313,7 @@ def test_show_config(self):
r"installpath\s* \(E\) = " + os.path.join(self.test_prefix, 'tmp.*'),
r"repositorypath\s* \(D\) = " + os.path.join(default_prefix, 'ebfiles_repo'),
r"robot-paths\s* \(E\) = " + os.path.join(test_dir, 'easyconfigs', 'test_ecs'),
+ r"rpath\s* \(D\) = " + ('False' if get_os_type() == DARWIN else 'True'),
r"sourcepath\s* \(E\) = " + os.path.join(test_dir, 'sandbox', 'sources'),
r"subdir-modules\s* \(F\) = mods",
]
@@ -5072,24 +5362,23 @@ def test_show_config_cfg_levels(self):
"""Test --show-config in relation to how configuring across multiple configuration levels interacts with it."""
# make sure default module syntax is used
- if 'EASYBUILD_MODULE_SYNTAX' in os.environ:
- del os.environ['EASYBUILD_MODULE_SYNTAX']
+ os.environ.pop('EASYBUILD_MODULE_SYNTAX', None)
# configuring --modules-tool and --module-syntax on different levels should NOT cause problems
# cfr. bug report https://github.com/easybuilders/easybuild-framework/issues/2564
- os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModulesC'
+ os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModules'
args = [
'--module-syntax=Tcl',
'--show-config',
]
# set init_config to False to avoid that eb_main (called by _run_mock_eb) re-initialises configuration
- # this fails because $EASYBUILD_MODULES_TOOL=EnvironmentModulesC conflicts with default module syntax (Lua)
+ # this fails because $EASYBUILD_MODULES_TOOL=EnvironmentModules conflicts with default module syntax (Lua)
stdout, _ = self._run_mock_eb(args, raise_error=True, redo_init_config=False)
patterns = [
r"^# Current EasyBuild configuration",
r"^module-syntax\s*\(C\) = Tcl",
- r"^modules-tool\s*\(E\) = EnvironmentModulesC",
+ r"^modules-tool\s*\(E\) = EnvironmentModules",
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
@@ -5099,11 +5388,10 @@ def test_modules_tool_vs_syntax_check(self):
"""Verify that check for modules tool vs syntax works."""
# make sure default module syntax is used
- if 'EASYBUILD_MODULE_SYNTAX' in os.environ:
- del os.environ['EASYBUILD_MODULE_SYNTAX']
+ os.environ.pop('EASYBUILD_MODULE_SYNTAX', None)
- # using EnvironmentModulesC modules tool with default module syntax (Lua) is a problem
- os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModulesC'
+ # using EnvironmentModules modules tool with default module syntax (Lua) is a problem
+ os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModules'
args = ['--show-full-config']
error_pattern = "Generating Lua module files requires Lmod as modules tool"
self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, args, raise_error=True)
@@ -5111,10 +5399,10 @@ def test_modules_tool_vs_syntax_check(self):
patterns = [
r"^# Current EasyBuild configuration",
r"^module-syntax\s*\(C\) = Tcl",
- r"^modules-tool\s*\(E\) = EnvironmentModulesC",
+ r"^modules-tool\s*\(E\) = EnvironmentModules",
]
- # EnvironmentModulesC modules tool + Tcl module syntax is fine
+ # EnvironmentModules modules tool + Tcl module syntax is fine
args.append('--module-syntax=Tcl')
stdout, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, redo_init_config=False)
for pattern in patterns:
@@ -5192,6 +5480,45 @@ def test_dump_env_script(self):
])
self.assertEqual(res.output.strip(), expected_out)
+ def test_dump_env_script_existing_module(self):
+ toy_ec = 'toy-0.0.eb'
+
+ os.chdir(self.test_prefix)
+ self._run_mock_eb([toy_ec, '--force'], do_build=True)
+ env_script = os.path.join(self.test_prefix, os.path.splitext(toy_ec)[0] + '.env')
+ test_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
+ if get_module_syntax() == 'Lua':
+ test_module += '.lua'
+ self.assertExists(test_module)
+ self.assertNotExists(env_script)
+
+ args = [toy_ec, '--dump-env']
+ os.chdir(self.test_prefix)
+ self._run_mock_eb(args, do_build=True, raise_error=True)
+ self.assertExists(env_script)
+ self.assertExists(test_module)
+ module_content = read_file(test_module)
+ env_file_content = read_file(env_script)
+
+ error_msg = (r"Script\(s\) already exists, not overwriting them \(unless --force is used\): "
+ + os.path.basename(env_script))
+ os.chdir(self.test_prefix)
+ self.assertErrorRegex(EasyBuildError, error_msg, self._run_mock_eb, args, do_build=True, raise_error=True)
+ self.assertExists(env_script)
+ self.assertExists(test_module)
+ # Unchanged module and env file
+ self.assertEqual(read_file(test_module), module_content)
+ self.assertEqual(read_file(env_script), env_file_content)
+
+ args.append('--force')
+ os.chdir(self.test_prefix)
+ self._run_mock_eb(args, do_build=True, raise_error=True)
+ self.assertExists(env_script)
+ self.assertExists(test_module)
+ # Unchanged module and env file
+ self.assertEqual(read_file(test_module), module_content)
+ self.assertEqual(read_file(env_script), env_file_content)
+
def test_stop(self):
"""Test use of --stop."""
args = ['toy-0.0.eb', '--force', '--stop=configure']
@@ -5221,7 +5548,7 @@ def test_fetch(self):
# which might trip up the dependency resolution (see #4298)
for ec in ('toy-0.0.eb', 'toy-0.0-deps.eb'):
args = [ec, '--fetch']
- stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False)
+ stdout, _ = self._run_mock_eb(args, raise_error=True, strip=True, testing=False)
patterns = [
r"^== fetching files\.\.\.$",
@@ -5234,6 +5561,17 @@ def test_fetch(self):
regex = re.compile(r"^== creating build dir, resetting environment\.\.\.$")
self.assertFalse(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+ # --fetch should also verify the checksums
+ tmpdir = tempfile.mkdtemp(prefix='easybuild-sources')
+ write_file(os.path.join(tmpdir, 'toy-0.0.tar.gz'), 'Make checksum check fail')
+ args = ['--sourcepath=%s:%s' % (tmpdir, self.test_sourcepath), '--fetch', 'toy-0.0.eb']
+ with self.mocked_stdout_stderr():
+ pattern = 'Checksum verification for .*/toy-0.0.tar.gz .*failed'
+ self.assertErrorRegex(EasyBuildError, pattern, self.eb_main, args, do_build=True, raise_error=True)
+ # We can avoid that failure by ignoring the checksums
+ args.append('--ignore-checksums')
+ self.eb_main(args, do_build=True, raise_error=True)
+
def test_parse_external_modules_metadata(self):
"""Test parse_external_modules_metadata function."""
# by default, provided external module metadata cfg files are picked up
@@ -5582,14 +5920,9 @@ def test_parse_optarch(self):
def test_check_contrib_style(self):
"""Test style checks performed by --check-contrib + dedicated --check-style option."""
- try:
- import pycodestyle # noqa
- except ImportError:
- try:
- import pep8 # noqa
- except ImportError:
- print("Skipping test_check_contrib_style, since pycodestyle or pep8 is not available")
- return
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_check_contrib_style pycodestyle is not available")
+ return
regex = re.compile(r"Running style check on 2 easyconfig\(s\)(.|\n)*>> All style checks PASSed!", re.M)
args = [
@@ -5642,8 +5975,8 @@ def test_check_contrib_style(self):
def test_check_contrib_non_style(self):
"""Test non-style checks performed by --check-contrib."""
- if not ('pycodestyle' in sys.modules or 'pep8' in sys.modules):
- print("Skipping test_check_contrib_non_style (no pycodestyle or pep8 available)")
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_check_contrib_non_style pycodestyle is not available")
return
args = [
@@ -5898,7 +6231,7 @@ def test_inject_checksums(self):
patterns = [
r"^== injecting sha256 checksums in .*/test\.eb$",
r"^== fetching sources & patches for test\.eb\.\.\.$",
- r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+\.\.\.$",
+ r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+$",
r"^== injecting sha256 checksums for sources & patches in test\.eb\.\.\.$",
r"^== \* toy-0.0\.tar\.gz: %s$" % toy_source_sha256,
r"^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: %s$" % toy_patch_sha256,
@@ -6019,18 +6352,19 @@ def test_inject_checksums(self):
self.assertNotIn('checksums = ', toy_ec_txt)
write_file(test_ec, toy_ec_txt)
- args = [test_ec, '--inject-checksums=md5']
+ args = [test_ec, '--inject-checksums=sha256']
stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True)
patterns = [
- r"^== injecting md5 checksums in .*/test\.eb$",
+ r"^== injecting sha256 checksums in .*/test\.eb$",
r"^== fetching sources & patches for test\.eb\.\.\.$",
- r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+\.\.\.$",
- r"^== injecting md5 checksums for sources & patches in test\.eb\.\.\.$",
- r"^== \* toy-0.0\.tar\.gz: be662daa971a640e40be5c804d9d7d10$",
- r"^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: a99f2a72cee1689a2f7e3ace0356efb1$",
- r"^== \* toy-extra\.txt: 3b0787b3bf36603ae1398c4a49097893$",
+ r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+$",
+ r"^== injecting sha256 checksums for sources & patches in test\.eb\.\.\.$",
+ r"^== \* toy-0.0\.tar\.gz: 44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc$",
+ r"^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: " # no comma, continues on next line
+ r"81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487$",
+ r"^== \* toy-extra\.txt: 4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458$",
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
@@ -6046,9 +6380,10 @@ def test_inject_checksums(self):
# no parse errors for updated easyconfig file...
ec = EasyConfigParser(test_ec).get_config_dict()
checksums = [
- {'toy-0.0.tar.gz': 'be662daa971a640e40be5c804d9d7d10'},
- {'toy-0.0_fix-silly-typo-in-printf-statement.patch': 'a99f2a72cee1689a2f7e3ace0356efb1'},
- {'toy-extra.txt': '3b0787b3bf36603ae1398c4a49097893'},
+ {'toy-0.0.tar.gz': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'},
+ {'toy-0.0_fix-silly-typo-in-printf-statement.patch':
+ '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'},
+ {'toy-extra.txt': '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458'},
]
self.assertEqual(ec['checksums'], checksums)
@@ -6632,6 +6967,23 @@ def test_sysroot(self):
os.environ['EASYBUILD_SYSROOT'] = doesnotexist
self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True)
+ def test_software_commit(self):
+ """Test use of --software-commit option."""
+
+ software_commit = "23be34"
+ software_commit_arg = '--software-commit=' + software_commit
+ # Add robot to also test that it gets disabled
+ stdout, stderr = self._run_mock_eb([software_commit_arg, '--show-config', '--robot'], raise_error=True)
+
+ warning_regex = re.compile(r'.*WARNING:.*--software-commit robot resolution is being disabled.*', re.M)
+ software_commit_regex = re.compile(r'^software-commit\s*\(C\) = %s$' % software_commit, re.M)
+ robot_regex = re.compile(r'^robot\s*\(C\) = .*', re.M)
+
+ self.assertTrue(warning_regex.search(stderr), "Pattern '%s' not found in: %s" % (warning_regex, stderr))
+ self.assertTrue(software_commit_regex.search(stdout),
+ "Pattern '%s' not found in: %s" % (software_commit_regex, stdout))
+ self.assertFalse(robot_regex.search(stdout), "Pattern '%s' found in: %s" % (robot_regex, stdout))
+
def test_accept_eula_for(self):
"""Test --accept-eula-for configuration option."""
@@ -6839,8 +7191,7 @@ def test_easystack_opts(self):
mod_ext = '.lua' if get_module_syntax() == 'Lua' else ''
# make sure that $EBROOTLIBTOY is not defined
- if 'EBROOTLIBTOY' in os.environ:
- del os.environ['EBROOTLIBTOY']
+ os.environ.pop('EBROOTLIBTOY', None)
# libtoy module should be installed, module file should at least set EBROOTLIBTOY
mod_dir = os.path.join(self.test_installpath, 'modules', 'all')
@@ -7011,6 +7362,15 @@ def test_opts_dict_to_eb_opts(self):
]
self.assertEqual(opts_dict_to_eb_opts(opts_dict), expected)
+ # multi-call options
+ opts_dict = {'try-amend': ['a=1', 'b=2', 'c=3']}
+ expected = ['--try-amend=a=1', '--try-amend=b=2', '--try-amend=c=3']
+ self.assertEqual(opts_dict_to_eb_opts(opts_dict), expected)
+
+ opts_dict = {'amend': ['a=1', 'b=2', 'c=3']}
+ expected = ['--amend=a=1', '--amend=b=2', '--amend=c=3']
+ self.assertEqual(opts_dict_to_eb_opts(opts_dict), expected)
+
def suite():
""" returns all the testcases in this module """
diff --git a/test/framework/output.py b/test/framework/output.py
index d1673a55d1..7e32629995 100644
--- a/test/framework/output.py
+++ b/test/framework/output.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2021-2023 Ghent University
+# Copyright 2021-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -35,7 +35,7 @@
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option, get_output_style, update_build_option
from easybuild.tools.output import PROGRESS_BAR_EXTENSIONS, PROGRESS_BAR_TYPES
-from easybuild.tools.output import DummyRich, colorize, get_progress_bar, show_progress_bars
+from easybuild.tools.output import DummyRich, colorize, get_progress_bar, print_error, show_progress_bars
from easybuild.tools.output import start_progress_bar, status_bar, stop_progress_bar, update_progress_bar, use_rich
try:
@@ -139,6 +139,26 @@ def test_colorize(self):
self.assertErrorRegex(EasyBuildError, "Unknown color: nosuchcolor", colorize, 'test', 'nosuchcolor')
+ def test_print_error(self):
+ """
+ Test print_error function
+ """
+ msg = "This is yellow: " + colorize("a banana", color='yellow')
+ self.mock_stderr(True)
+ self.mock_stdout(True)
+ print_error(msg)
+ stderr = self.get_stderr()
+ stdout = self.get_stdout()
+ self.mock_stderr(False)
+ self.mock_stdout(False)
+ self.assertEqual(stdout, '')
+ if HAVE_RICH:
+ # when using Rich, message printed to stderr won't have funny terminal escape characters for the color
+ expected = '\n\nThis is yellow: a banana\n\n'
+ else:
+ expected = '\nThis is yellow: \x1b[1;33ma banana\x1b[0m\n\n'
+ self.assertEqual(stderr, expected)
+
def test_get_progress_bar(self):
"""
Test get_progress_bar.
diff --git a/test/framework/package.py b/test/framework/package.py
index 7298d6ecc9..72b14629bc 100644
--- a/test/framework/package.py
+++ b/test/framework/package.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py
index 87f62fbab8..975e46375b 100644
--- a/test/framework/parallelbuild.py
+++ b/test/framework/parallelbuild.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -223,6 +223,8 @@ def test_build_easyconfigs_in_parallel_gc3pie(self):
print("GC3Pie not available, skipping test")
return
+ self.allow_deprecated_behaviour()
+
# put GC3Pie config in place to use local host and fork/exec
resourcedir = os.path.join(self.test_prefix, 'gc3pie')
gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini')
@@ -262,7 +264,9 @@ def test_build_easyconfigs_in_parallel_gc3pie(self):
topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path)
- build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
+
+ with self.mocked_stdout_stderr():
+ build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
toy_modfile = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
if get_module_syntax() == 'Lua':
@@ -280,7 +284,8 @@ def test_build_easyconfigs_in_parallel_gc3pie(self):
ecs = resolve_dependencies(process_easyconfig(test_ecfile), self.modtool)
error = "1 jobs failed: toy-1.2.3"
- self.assertErrorRegex(EasyBuildError, error, build_easyconfigs_in_parallel, cmd, ecs, prepare_first=False)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error, build_easyconfigs_in_parallel, cmd, ecs, prepare_first=False)
def test_submit_jobs(self):
"""Test submit_jobs"""
diff --git a/test/framework/repository.py b/test/framework/repository.py
index acc2749c55..b45d5d7165 100644
--- a/test/framework/repository.py
+++ b/test/framework/repository.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,7 +40,6 @@
from easybuild.tools.filetools import read_file
from easybuild.tools.repository.filerepo import FileRepository
from easybuild.tools.repository.gitrepo import GitRepository
-from easybuild.tools.repository.hgrepo import HgRepository
from easybuild.tools.repository.svnrepo import SvnRepository
from easybuild.tools.repository.repository import init_repository
from easybuild.tools.run import run_shell_cmd
@@ -129,27 +128,6 @@ def test_svnrepo(self):
self.assertExists(os.path.join(repo.wc, 'trunk', 'README.md'))
shutil.rmtree(repo.wc)
- # this test is disabled because it fails in Travis as a result of bitbucket disabling TLS 1.0/1.1
- # we can consider re-enabling it when moving to a more recent Ubuntu version in the Travis config
- # (which implies dropping support for Python 2.6)
- # cfr. https://github.com/easybuilders/easybuild-framework/pull/2678
- def DISABLED_test_hgrepo(self):
- """Test using HgRepository."""
- # only run this test if pysvn Python module is available
- try:
- import hglib # noqa
- except ImportError:
- print("(skipping HgRepository test)")
- return
-
- # GitHub also supports SVN
- test_repo_url = 'https://kehoste@bitbucket.org/kehoste/testrepository'
-
- repo = HgRepository(test_repo_url)
- repo.init()
- self.assertExists(os.path.join(repo.wc, 'README'))
- shutil.rmtree(repo.wc)
-
def test_init_repository(self):
"""Test use of init_repository function."""
repo = init_repository('FileRepository', self.path)
diff --git a/test/framework/robot.py b/test/framework/robot.py
index f52e9b2c5c..62350bfae4 100644
--- a/test/framework/robot.py
+++ b/test/framework/robot.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -128,8 +128,7 @@ def tearDown(self):
config.modules_tool = ORIG_MODULES_TOOL
ecec.modules_tool = ORIG_ECEC_MODULES_TOOL
if ORIG_MODULE_FUNCTION is None:
- if 'module' in os.environ:
- del os.environ['module']
+ os.environ.pop('module', None)
else:
os.environ['module'] = ORIG_MODULE_FUNCTION
self.modtool = self.orig_modtool
@@ -713,6 +712,47 @@ def test_search_paths(self):
regex = re.compile(r"^ \* %s$" % os.path.join(self.test_prefix, test_ec), re.M)
self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
+ def test_github_det_easyconfig_paths_from_commit(self):
+ """Test det_easyconfig_paths function in combination with --from-commit."""
+ # note: --from-commit does not involve using GitHub API, so no GitHub token required
+
+ test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+
+ commit = '589282cf52609067616fc2a522f8e4b81f809cb7'
+ args = [
+ os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), # absolute path
+ 'toy-0.0-iter.eb', # relative path, available via robot search path
+ # commit in which ReFrame-4.3.2.eb was added,
+ # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18763/commits
+ '--from-commit', commit,
+ 'ReFrame-4.3.2.eb', # easyconfig included in commit, should be resolved via robot search path
+ '--dry-run',
+ '--robot',
+ '--robot=%s' % test_ecs_path,
+ '--unittest-file=%s' % self.logfile,
+ '--tmpdir=%s' % self.test_prefix,
+ ]
+
+ self.mock_stderr(True)
+ outtxt = self.eb_main(args, raise_error=True)
+ stderr = self.get_stderr()
+ self.mock_stderr(False)
+
+ self.assertFalse(stderr)
+
+ # full path doesn't matter (helps to avoid failing tests due to resolved symlinks)
+ test_ecs_path = os.path.join('.*', 'test', 'framework', 'easyconfigs', 'test_ecs')
+
+ modules = [
+ (test_ecs_path, 'toy/0.0'),
+ (test_ecs_path, 'toy/0.0-iter'),
+ (os.path.join(self.test_prefix, '.*', 'files_commit_%s' % commit), 'ReFrame/4.3.2'),
+ ]
+ for path_prefix, module in modules:
+ ec_fn = "%s.eb" % '-'.join(module.split('/'))
+ regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path_prefix, ec_fn, module), re.M)
+ self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
+
def test_github_det_easyconfig_paths_from_pr(self):
"""Test det_easyconfig_paths function, with --from-pr enabled as well."""
if self.github_token is None:
@@ -1084,8 +1124,8 @@ def test_tweak_robotpath(self):
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
# Create directories to store the tweaked easyconfigs
- tweaked_ecs_paths, pr_paths = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True)
- robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, pr_paths, auto_robot=True)
+ tweaked_ecs_paths, extra_ec_paths = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True)
+ robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, extra_ec_paths, auto_robot=True)
init_config(build_options={
'valid_module_classes': module_classes(),
diff --git a/test/framework/run.py b/test/framework/run.py
index 2f16e3978a..765efc6b62 100644
--- a/test/framework/run.py
+++ b/test/framework/run.py
@@ -1,6 +1,6 @@
# #
# -*- coding: utf-8 -*-
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -24,7 +24,7 @@
# along with EasyBuild. If not, see .
# #
"""
-Unit tests for filetools.py
+Unit tests for run.py
@author: Toon Willems (Ghent University)
@author: Kenneth Hoste (Ghent University)
@@ -44,14 +44,15 @@
import time
from concurrent.futures import ThreadPoolExecutor
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
-from unittest import TextTestRunner
+from unittest import TextTestRunner, mock
from easybuild.base.fancylogger import setLogLevelDebug
import easybuild.tools.asyncprocess as asyncprocess
import easybuild.tools.utilities
from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging
from easybuild.tools.config import update_build_option
-from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, write_file
+from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, remove_dir, write_file
+from easybuild.tools.modules import EnvironmentModules, Lmod
from easybuild.tools.run import RunShellCmdResult, RunShellCmdError, check_async_cmd, check_log_for_errors
from easybuild.tools.run import complete_cmd, fileprefix_from_cmd, get_output_from_process, parse_log_for_error
from easybuild.tools.run import run_cmd, run_cmd_qa, run_shell_cmd, subprocess_terminate
@@ -76,6 +77,9 @@ def tearDown(self):
def test_get_output_from_process(self):
"""Test for get_output_from_process utility function."""
+ # use of get_output_from_process is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
@contextlib.contextmanager
def get_proc(cmd, asynchronous=False):
if asynchronous:
@@ -92,56 +96,65 @@ def get_proc(cmd, asynchronous=False):
subprocess_terminate(proc, timeout=1)
# get all output at once
- with get_proc("echo hello") as proc:
- out = get_output_from_process(proc)
- self.assertEqual(out, 'hello\n')
+ with self.mocked_stdout_stderr():
+ with get_proc("echo hello") as proc:
+ out = get_output_from_process(proc)
+ self.assertEqual(out, 'hello\n')
# first get 100 bytes, then get the rest all at once
- with get_proc("echo hello") as proc:
- out = get_output_from_process(proc, read_size=100)
- self.assertEqual(out, 'hello\n')
- out = get_output_from_process(proc)
- self.assertEqual(out, '')
+ with self.mocked_stdout_stderr():
+ with get_proc("echo hello") as proc:
+ out = get_output_from_process(proc, read_size=100)
+ self.assertEqual(out, 'hello\n')
+ out = get_output_from_process(proc)
+ self.assertEqual(out, '')
# get output in small bits, keep trying to get output (which shouldn't fail)
- with get_proc("echo hello") as proc:
- out = get_output_from_process(proc, read_size=1)
- self.assertEqual(out, 'h')
- out = get_output_from_process(proc, read_size=3)
- self.assertEqual(out, 'ell')
- out = get_output_from_process(proc, read_size=2)
- self.assertEqual(out, 'o\n')
- out = get_output_from_process(proc, read_size=1)
- self.assertEqual(out, '')
- out = get_output_from_process(proc, read_size=10)
- self.assertEqual(out, '')
- out = get_output_from_process(proc)
- self.assertEqual(out, '')
+ with self.mocked_stdout_stderr():
+ with get_proc("echo hello") as proc:
+ out = get_output_from_process(proc, read_size=1)
+ self.assertEqual(out, 'h')
+ out = get_output_from_process(proc, read_size=3)
+ self.assertEqual(out, 'ell')
+ out = get_output_from_process(proc, read_size=2)
+ self.assertEqual(out, 'o\n')
+ out = get_output_from_process(proc, read_size=1)
+ self.assertEqual(out, '')
+ out = get_output_from_process(proc, read_size=10)
+ self.assertEqual(out, '')
+ out = get_output_from_process(proc)
+ self.assertEqual(out, '')
# can also get output asynchronously (read_size is *ignored* in that case)
async_cmd = "echo hello; read reply; echo $reply"
- with get_proc(async_cmd, asynchronous=True) as proc:
- out = get_output_from_process(proc, asynchronous=True)
- self.assertEqual(out, 'hello\n')
- asyncprocess.send_all(proc, 'test123\n')
- out = get_output_from_process(proc)
- self.assertEqual(out, 'test123\n')
+ with self.mocked_stdout_stderr():
+ with get_proc(async_cmd, asynchronous=True) as proc:
+ out = get_output_from_process(proc, asynchronous=True)
+ self.assertEqual(out, 'hello\n')
+ asyncprocess.send_all(proc, 'test123\n')
+ out = get_output_from_process(proc)
+ self.assertEqual(out, 'test123\n')
- with get_proc(async_cmd, asynchronous=True) as proc:
- out = get_output_from_process(proc, asynchronous=True, read_size=1)
- # read_size is ignored when getting output asynchronously, we're getting more than 1 byte!
- self.assertEqual(out, 'hello\n')
- asyncprocess.send_all(proc, 'test123\n')
- out = get_output_from_process(proc, read_size=3)
- self.assertEqual(out, 'tes')
- out = get_output_from_process(proc, read_size=2)
- self.assertEqual(out, 't1')
- out = get_output_from_process(proc)
- self.assertEqual(out, '23\n')
+ with self.mocked_stdout_stderr():
+ with get_proc(async_cmd, asynchronous=True) as proc:
+ out = get_output_from_process(proc, asynchronous=True, read_size=1)
+ # read_size is ignored when getting output asynchronously, we're getting more than 1 byte!
+ self.assertEqual(out, 'hello\n')
+ asyncprocess.send_all(proc, 'test123\n')
+ out = get_output_from_process(proc, read_size=3)
+ self.assertEqual(out, 'tes')
+ out = get_output_from_process(proc, read_size=2)
+ self.assertEqual(out, 't1')
+ out = get_output_from_process(proc)
+ self.assertEqual(out, '23\n')
def test_run_cmd(self):
"""Basic test for run_cmd function."""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
with self.mocked_stdout_stderr():
(out, ec) = run_cmd("echo hello")
self.assertEqual(out, "hello\n")
@@ -153,7 +166,7 @@ def test_run_cmd(self):
# this is constructed to reproduce errors like:
# UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2
# UnicodeEncodeError: 'ascii' codec can't encode character u'\u2018'
- for text in [b"foo \xe2 bar", b"foo \u2018 bar"]:
+ for text in [b"foo \xe2 bar", "foo \u2018 bar"]:
test_file = os.path.join(self.test_prefix, 'foo.txt')
write_file(test_file, text)
cmd = "cat %s" % test_file
@@ -167,6 +180,10 @@ def test_run_cmd(self):
def test_run_shell_cmd_basic(self):
"""Basic test for run_shell_cmd function."""
+ os.environ['FOOBAR'] = 'foobar'
+
+ cwd = change_dir(self.test_prefix)
+
with self.mocked_stdout_stderr():
res = run_shell_cmd("echo hello")
self.assertEqual(res.output, "hello\n")
@@ -177,12 +194,66 @@ def test_run_shell_cmd_basic(self):
self.assertEqual(res.stderr, None)
self.assertTrue(res.work_dir and isinstance(res.work_dir, str))
+ change_dir(cwd)
+ del os.environ['FOOBAR']
+
+ # check on helper scripts that were generated for this command
+ paths = glob.glob(os.path.join(self.test_prefix, 'eb-*', 'run-shell-cmd-output', 'echo-*'))
+ self.assertEqual(len(paths), 1)
+ cmd_tmpdir = paths[0]
+
+ # check on env.sh script that can be used to set up environment in which command was run
+ env_script = os.path.join(cmd_tmpdir, 'env.sh')
+ self.assertExists(env_script)
+ env_script_txt = read_file(env_script)
+ self.assertIn("export FOOBAR=foobar", env_script_txt)
+ self.assertIn("history -s 'echo hello'", env_script_txt)
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"source {env_script}; echo $USER; echo $FOOBAR; history")
+ self.assertEqual(res.exit_code, 0)
+ user = os.getenv('USER')
+ self.assertTrue(res.output.startswith(f'{user}\nfoobar\n'))
+ self.assertTrue(res.output.endswith("echo hello\n"))
+
+ # check on cmd.sh script that can be used to create interactive shell environment for command
+ cmd_script = os.path.join(cmd_tmpdir, 'cmd.sh')
+ self.assertExists(cmd_script)
+
+ cmd = f"{cmd_script} -c 'echo pwd: $PWD; echo $FOOBAR; echo $EB_CMD_OUT_FILE; cat $EB_CMD_OUT_FILE'"
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, fail_on_error=False)
+ self.assertEqual(res.exit_code, 0)
+ regex = re.compile("pwd: .*\nfoobar\n.*/echo-.*/out.txt\nhello$")
+ self.assertTrue(regex.search(res.output), f"Pattern '{regex.pattern}' should be found in {res.output}")
+
+ # check whether working directory is what's expected
+ regex = re.compile('^pwd: .*', re.M)
+ res = regex.findall(res.output)
+ self.assertEqual(len(res), 1)
+ pwd = res[0].strip()[5:]
+ self.assertTrue(os.path.samefile(pwd, self.test_prefix))
+
+ cmd = f"{cmd_script} -c 'module --version'"
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, fail_on_error=False)
+ self.assertEqual(res.exit_code, 0)
+
+ if isinstance(self.modtool, Lmod):
+ regex = re.compile("^Modules based on Lua: Version [0-9]", re.M)
+ elif isinstance(self.modtool, EnvironmentModules):
+ regex = re.compile("^Modules Release [0-9]", re.M)
+ else:
+ self.fail("Unknown modules tool used!")
+
+ self.assertTrue(regex.search(res.output), f"Pattern '{regex.pattern}' should be found in {res.output}")
+
# test running command that emits non-UTF-8 characters
# this is constructed to reproduce errors like:
# UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2
# UnicodeEncodeError: 'ascii' codec can't encode character u'\u2018'
# (such errors are ignored by the 'run' implementation)
- for text in [b"foo \xe2 bar", b"foo \u2018 bar"]:
+ for text in [b"foo \xe2 bar", "foo \u2018 bar"]:
test_file = os.path.join(self.test_prefix, 'foo.txt')
write_file(test_file, text)
cmd = "cat %s" % test_file
@@ -195,6 +266,46 @@ def test_run_shell_cmd_basic(self):
self.assertTrue(isinstance(res.output, str))
self.assertTrue(res.work_dir and isinstance(res.work_dir, str))
+ def test_run_shell_cmd_env(self):
+ """Test env option in run_shell_cmd."""
+
+ # use 'env' to define environment in which command should be run;
+ # with a few exceptions (like $_, $PWD) no other environment variables will be defined,
+ # so $HOME and $USER will not be set
+ cmd = "env | sort"
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, env={'FOOBAR': 'foobar', 'PATH': os.getenv('PATH')})
+ self.assertEqual(res.cmd, cmd)
+ self.assertEqual(res.exit_code, 0)
+ self.assertIn("FOOBAR=foobar\n", res.output)
+ self.assertTrue(re.search("^_=.*/env$", res.output, re.M))
+ for var in ('HOME', 'USER'):
+ self.assertFalse(re.search('^' + var + '=.*', res.output, re.M))
+
+ # check on helper scripts that were generated for this command
+ paths = glob.glob(os.path.join(self.test_prefix, 'eb-*', 'run-shell-cmd-output', 'env-*'))
+ self.assertEqual(len(paths), 1)
+ cmd_tmpdir = paths[0]
+
+ # set environment variable in current environment,
+ # this should not be set in shell environment produced by scripts
+ os.environ['TEST123'] = 'test123'
+
+ env_script = os.path.join(cmd_tmpdir, 'env.sh')
+ self.assertExists(env_script)
+ env_script_txt = read_file(env_script)
+ self.assertIn('unset "$var"', env_script_txt)
+ self.assertIn('unset -f "$func"', env_script_txt)
+ self.assertIn('\nexport FOOBAR=foobar\nexport PATH', env_script_txt)
+
+ cmd_script = os.path.join(cmd_tmpdir, 'cmd.sh')
+ self.assertExists(cmd_script)
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(f"{cmd_script} -c 'echo $FOOBAR; echo TEST123:$TEST123'", fail_on_error=False)
+ self.assertEqual(res.exit_code, 0)
+ self.assertTrue(res.output.endswith('\nfoobar\nTEST123:\n'))
+
def test_fileprefix_from_cmd(self):
"""test simplifications from fileprefix_from_cmd."""
cmds = {
@@ -217,6 +328,10 @@ def test_fileprefix_from_cmd(self):
def test_run_cmd_log(self):
"""Test logging of executed commands."""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
os.close(fd)
@@ -300,6 +415,10 @@ def test_run_shell_cmd_log(self):
def test_run_cmd_negative_exit_code(self):
"""Test run_cmd function with command that has negative exit code."""
+
+ # use of run_cmd/run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
# define signal handler to call in case run_cmd takes too long
def handler(signum, _):
raise RuntimeError("Signal handler called with signal %s" % signum)
@@ -373,12 +492,14 @@ def handler(signum, _):
# check error reporting output
stderr = stderr.getvalue()
patterns = [
- r"^ERROR: Shell command failed!",
- r"^\s+full command\s* -> kill -9 \$\$",
- r"^\s+exit code\s* -> -9",
- r"^\s+working directory\s* -> " + work_dir,
- r"^\s+called from\s* -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)",
- r"^\s+output \(stdout \+ stderr\)\s* -> .*/run-shell-cmd-output/kill-.*/out.txt",
+ r"ERROR: Shell command failed!",
+ r"\s+full command\s* -> kill -9 \$\$",
+ r"\s+exit code\s* -> -9",
+ r"\s+working directory\s* -> " + work_dir,
+ r"\s+called from\s* -> 'test_run_shell_cmd_fail' function in "
+ r"(.|\n)*/test/(.|\n)*/run.py \(line [0-9]+\)",
+ r"\s+output \(stdout \+ stderr\)\s* -> (.|\n)*/run-shell-cmd-output/kill-(.|\n)*/out.txt",
+ r"\s+interactive shell script\s* -> (.|\n)*/run-shell-cmd-output/kill-(.|\n)*/cmd.sh",
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
@@ -408,13 +529,15 @@ def handler(signum, _):
# check error reporting output
stderr = stderr.getvalue()
patterns = [
- r"^ERROR: Shell command failed!",
- r"^\s+full command\s+ -> kill -9 \$\$",
- r"^\s+exit code\s+ -> -9",
- r"^\s+working directory\s+ -> " + work_dir,
- r"^\s+called from\s+ -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)",
- r"^\s+output \(stdout\)\s+ -> .*/run-shell-cmd-output/kill-.*/out.txt",
- r"^\s+error/warnings \(stderr\)\s+ -> .*/run-shell-cmd-output/kill-.*/err.txt",
+ r"ERROR: Shell command failed!",
+ r"\s+full command\s+ -> kill -9 \$\$",
+ r"\s+exit code\s+ -> -9",
+ r"\s+working directory\s+ -> " + work_dir,
+ r"\s+called from\s+ -> 'test_run_shell_cmd_fail' function in "
+ r"(.|\n)*/test/(.|\n)*/run.py \(line [0-9]+\)",
+ r"\s+output \(stdout\)\s+ -> (.|\n)*/run-shell-cmd-output/kill-(.|\n)*/out.txt",
+ r"\s+error/warnings \(stderr\)\s+ -> (.|\n)*/run-shell-cmd-output/kill-(.|\n)*/err.txt",
+ r"\s+interactive shell script\s* -> (.|\n)*/run-shell-cmd-output/kill-(.|\n)*/cmd.sh",
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
@@ -433,6 +556,10 @@ def handler(signum, _):
def test_run_cmd_bis(self):
"""More 'complex' test for run_cmd function."""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
# a more 'complex' command to run, make sure all required output is there
with self.mocked_stdout_stderr():
(out, ec) = run_cmd("for j in `seq 1 3`; do for i in `seq 1 100`; do echo hello; done; sleep 1.4; done")
@@ -453,6 +580,10 @@ def test_run_cmd_work_dir(self):
"""
Test running command in specific directory with run_cmd function.
"""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
orig_wd = os.getcwd()
self.assertFalse(os.path.samefile(orig_wd, self.test_prefix))
@@ -472,27 +603,47 @@ def test_run_shell_cmd_work_dir(self):
"""
Test running shell command in specific directory with run_shell_cmd function.
"""
- orig_wd = os.getcwd()
- self.assertFalse(os.path.samefile(orig_wd, self.test_prefix))
-
test_dir = os.path.join(self.test_prefix, 'test')
+ test_workdir = os.path.join(self.test_prefix, 'test', 'workdir')
for fn in ('foo.txt', 'bar.txt'):
- write_file(os.path.join(test_dir, fn), 'test')
+ write_file(os.path.join(test_workdir, fn), 'test')
+
+ os.chdir(test_dir)
+ orig_wd = os.getcwd()
+ self.assertFalse(os.path.samefile(orig_wd, self.test_prefix))
cmd = "ls | sort"
+
+ # working directory is not explicitly defined
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd)
+
+ self.assertEqual(res.cmd, cmd)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, 'workdir\n')
+ self.assertEqual(res.stderr, None)
+ self.assertEqual(res.work_dir, orig_wd)
+
+ self.assertTrue(os.path.samefile(orig_wd, os.getcwd()))
+
+ # working directory is explicitly defined
with self.mocked_stdout_stderr():
- res = run_shell_cmd(cmd, work_dir=test_dir)
+ res = run_shell_cmd(cmd, work_dir=test_workdir)
self.assertEqual(res.cmd, cmd)
self.assertEqual(res.exit_code, 0)
self.assertEqual(res.output, 'bar.txt\nfoo.txt\n')
self.assertEqual(res.stderr, None)
- self.assertEqual(res.work_dir, test_dir)
+ self.assertEqual(res.work_dir, test_workdir)
self.assertTrue(os.path.samefile(orig_wd, os.getcwd()))
def test_run_cmd_log_output(self):
"""Test run_cmd with log_output enabled"""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
with self.mocked_stdout_stderr():
(out, ec) = run_cmd("seq 1 100", log_output=True)
self.assertEqual(ec, 0)
@@ -539,15 +690,42 @@ def test_run_shell_cmd_split_stderr(self):
self.assertTrue("warning" in output_lines)
self.assertEqual(res.stderr, None)
+ # cleanup of artifacts in between calls to run_shell_cmd
+ remove_dir(self.test_prefix)
+
with self.mocked_stdout_stderr():
res = run_shell_cmd(cmd, split_stderr=True)
self.assertEqual(res.exit_code, 0)
self.assertEqual(res.stderr, "warning\n")
self.assertEqual(res.output, "ok\n")
+ # check whether environment variables that point to stdout/stderr output files
+ # are set in environment defined by cmd.sh script
+ paths = glob.glob(os.path.join(self.test_prefix, 'eb-*', 'run-shell-cmd-output', 'echo-*'))
+ self.assertEqual(len(paths), 1)
+ cmd_tmpdir = paths[0]
+ cmd_script = os.path.join(cmd_tmpdir, 'cmd.sh')
+ self.assertExists(cmd_script)
+
+ cmd_cmd = '; '.join([
+ "echo $EB_CMD_OUT_FILE",
+ "cat $EB_CMD_OUT_FILE",
+ "echo $EB_CMD_ERR_FILE",
+ "cat $EB_CMD_ERR_FILE",
+ ])
+ cmd = f"{cmd_script} -c '{cmd_cmd}'"
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, fail_on_error=False)
+
+ regex = re.compile(".*/echo-.*/out.txt\nok\n.*/echo-.*/err.txt\nwarning$")
+ self.assertTrue(regex.search(res.output), f"Pattern '{regex.pattern}' should be found in {res.output}")
+
def test_run_cmd_trace(self):
"""Test run_cmd in trace mode, and with tracing disabled."""
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
pattern = [
r"^ >> running command:",
r"\t\[started at: .*\]",
@@ -567,7 +745,7 @@ def test_run_cmd_trace(self):
self.mock_stderr(False)
self.assertEqual(out, 'hello\n')
self.assertEqual(ec, 0)
- self.assertEqual(stderr, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
regex = re.compile('\n'.join(pattern))
self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
@@ -582,7 +760,7 @@ def test_run_cmd_trace(self):
self.mock_stderr(False)
self.assertEqual(out, 'hello\n')
self.assertEqual(ec, 0)
- self.assertEqual(stderr, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
self.assertEqual(stdout, '')
init_config(build_options={'trace': True})
@@ -597,7 +775,7 @@ def test_run_cmd_trace(self):
self.mock_stderr(False)
self.assertEqual(out, 'hello')
self.assertEqual(ec, 0)
- self.assertEqual(stderr, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
pattern.insert(3, r"\t\[input: hello\]")
pattern[-2] = "\tcat"
regex = re.compile('\n'.join(pattern))
@@ -614,7 +792,7 @@ def test_run_cmd_trace(self):
self.mock_stderr(False)
self.assertEqual(out, 'hello')
self.assertEqual(ec, 0)
- self.assertEqual(stderr, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
self.assertEqual(stdout, '')
# trace output can be disabled on a per-command basis
@@ -631,7 +809,7 @@ def test_run_cmd_trace(self):
self.assertEqual(out, 'hello\n')
self.assertEqual(ec, 0)
self.assertEqual(stdout, '')
- self.assertEqual(stderr, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
def test_run_shell_cmd_trace(self):
"""Test run_shell_cmd function in trace mode, and with tracing disabled."""
@@ -641,7 +819,7 @@ def test_run_shell_cmd_trace(self):
r"\techo hello",
r"\t\[started at: .*\]",
r"\t\[working dir: .*\]",
- r"\t\[output saved to .*\]",
+ r"\t\[output and state saved to .*\]",
r" >> command completed: exit 0, ran in .*",
]
@@ -701,7 +879,7 @@ def test_run_shell_cmd_trace_stdin(self):
r"\techo hello",
r"\t\[started at: [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\]",
r"\t\[working dir: .*\]",
- r"\t\[output saved to .*\]",
+ r"\t\[output and state saved to .*\]",
r" >> command completed: exit 0, ran in .*",
]
@@ -750,6 +928,9 @@ def test_run_shell_cmd_trace_stdin(self):
def test_run_cmd_qa(self):
"""Basic test for run_cmd_qa function."""
+ # use of run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
cmd = "echo question; read x; echo $x"
qa = {'question': 'answer'}
with self.mocked_stdout_stderr():
@@ -771,9 +952,171 @@ def test_run_cmd_qa(self):
self.assertTrue(out.startswith("question\nanswer\nfoo "))
self.assertTrue(out.endswith('bar'))
+ # test handling of output that is not actually a question
+ cmd = ';'.join([
+ "echo not-a-question-but-a-statement",
+ "sleep 3",
+ "echo question",
+ "read x",
+ "echo $x",
+ ])
+ qa = {'question': 'answer'}
+
+ # fails because non-question is encountered
+ error_pattern = "Max nohits 1 reached: end of output not-a-question-but-a-statement"
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd_qa, cmd, qa, maxhits=1, trace=False)
+
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd_qa(cmd, qa, no_qa=["not-a-question-but-a-statement"], maxhits=1, trace=False)
+ self.assertEqual(out, "not-a-question-but-a-statement\nquestion\nanswer\n")
+ self.assertEqual(ec, 0)
+
+ def test_run_shell_cmd_qa(self):
+ """Basic test for Q&A support in run_shell_cmd function."""
+
+ cmd = '; '.join([
+ "echo question1",
+ "read x",
+ "echo $x",
+ "echo question2",
+ "read y",
+ "echo $y",
+ ])
+ qa = [
+ ('question1', 'answer1'),
+ ('question2', 'answer2'),
+ ]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+ self.assertEqual(res.output, "question1\nanswer1\nquestion2\nanswer2\n")
+ # no reason echo hello could fail
+ self.assertEqual(res.exit_code, 0)
+
+ # test running command that emits non-UTF8 characters
+ # this is constructed to reproduce errors like:
+ # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2
+ test_file = os.path.join(self.test_prefix, 'foo.txt')
+ write_file(test_file, b"foo \xe2 bar")
+ cmd += "; cat %s" % test_file
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+ self.assertEqual(res.exit_code, 0)
+ self.assertTrue(res.output.startswith("question1\nanswer1\nquestion2\nanswer2\nfoo "))
+ self.assertTrue(res.output.endswith('bar'))
+
+ # check type check on qa_patterns
+ error_pattern = "qa_patterns passed to run_shell_cmd should be a list of 2-tuples!"
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns={'foo': 'bar'})
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=('foo', 'bar'))
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=(('foo', 'bar'),))
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns='foo:bar')
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=['foo:bar'])
+
+ # validate use of qa_timeout to give up if there's no matching question for too long
+ cmd = "sleep 3; echo 'question'; read a; echo $a"
+ error_pattern = "No matching questions found for current command output, giving up after 1 seconds!"
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=qa, qa_timeout=1)
+
+ # check using answer that is completed via pattern extracted from question
+ cmd = ';'.join([
+ "echo 'and the magic number is: 42'",
+ "read magic_number",
+ "echo $magic_number",
+ ])
+ qa = [("and the magic number is: (?P[0-9]+)", "%(nr)s")]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "and the magic number is: 42\n42\n")
+
+ # test handling of output that is not actually a question
+ cmd = ';'.join([
+ "echo not-a-question-but-a-statement",
+ "sleep 3",
+ "echo question",
+ "read x",
+ "echo $x",
+ ])
+ qa = [('question', 'answer')]
+
+ # fails because non-question is encountered
+ error_pattern = "No matching questions found for current command output, giving up after 1 seconds!"
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=qa, qa_timeout=1,
+ hidden=True)
+
+ qa_wait_patterns = ["not-a-question-but-a-statement"]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa, qa_wait_patterns=qa_wait_patterns, qa_timeout=1)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "not-a-question-but-a-statement\nquestion\nanswer\n")
+
+ # test multi-line question
+ cmd = ';'.join([
+ "echo please",
+ "echo answer",
+ "read x",
+ "echo $x",
+ ])
+ qa = [("please answer", "42")]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "please\nanswer\n42\n")
+
+ # also test multi-line wait pattern
+ cmd = "echo just; echo wait; sleep 3; " + cmd
+ qa_wait_patterns = ["just wait"]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa, qa_wait_patterns=qa_wait_patterns, qa_timeout=1)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "just\nwait\nplease\nanswer\n42\n")
+
+ # test multi-line question pattern with hard space
+ cmd = ';'.join([
+ "echo please",
+ "echo answer",
+ "read x",
+ "echo $x",
+ ])
+ # question pattern uses hard space, should get replaced internally by more liberal whitespace regex pattern
+ qa = [(r"please\ answer", "42")]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa, qa_timeout=3)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "please\nanswer\n42\n")
+
+ # test interactive command that takes a while before producing more output that includes second question
+ cmd = ';'.join([
+ "echo question1",
+ "read answer1",
+ "sleep 2",
+ "echo question2",
+ "read answer2",
+ # note: delaying additional output (except the actual questions) is important
+ # to verify that this is working as intended
+ "echo $answer1",
+ "echo $answer2",
+ ])
+ qa = [
+ (r'question1', 'answer1'),
+ (r'question2', 'answer2'),
+ ]
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "question1\nquestion2\nanswer1\nanswer2\n")
+
def test_run_cmd_qa_buffering(self):
"""Test whether run_cmd_qa uses unbuffered output."""
+ # use of run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
# command that generates a lot of output before waiting for input
# note: bug being fixed can be reproduced reliably using 1000, but not with too high values like 100000!
cmd = 'for x in $(seq 1000); do echo "This is a number you can pick: $x"; done; '
@@ -803,8 +1146,44 @@ def test_run_cmd_qa_buffering(self):
self.assertEqual(ec, 1)
self.assertEqual(out, "Hello, I am about to exit\nERROR: I failed\n")
+ def test_run_shell_cmd_qa_buffering(self):
+ """Test whether run_shell_cmd uses unbuffered output when running interactive commands."""
+
+ # command that generates a lot of output before waiting for input
+ # note: bug being fixed can be reproduced reliably using 1000, but not with too high values like 100000!
+ cmd = 'for x in $(seq 1000); do echo "This is a number you can pick: $x"; done; '
+ cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"'
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=[('Pick a number: ', '42')], qa_timeout=10)
+
+ self.assertEqual(res.exit_code, 0)
+ regex = re.compile("Picked number: 42$")
+ self.assertTrue(regex.search(res.output), f"Pattern '{regex.pattern}' found in: {res.output}")
+
+ # also test with script run as interactive command that quickly exits with non-zero exit code;
+ # see https://github.com/easybuilders/easybuild-framework/issues/3593
+ script_txt = '\n'.join([
+ "#/bin/bash",
+ "echo 'Hello, I am about to exit'",
+ "echo 'ERROR: I failed' >&2",
+ "exit 1",
+ ])
+ script = os.path.join(self.test_prefix, 'test.sh')
+ write_file(script, script_txt)
+ adjust_permissions(script, stat.S_IXUSR)
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(script, qa_patterns=[], fail_on_error=False)
+
+ self.assertEqual(res.exit_code, 1)
+ self.assertEqual(res.output, "Hello, I am about to exit\nERROR: I failed\n")
+
def test_run_cmd_qa_log_all(self):
"""Test run_cmd_qa with log_output enabled"""
+
+ # use of run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
with self.mocked_stdout_stderr():
(out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'}, log_all=True)
self.assertEqual(ec, 0)
@@ -816,13 +1195,25 @@ def test_run_cmd_qa_log_all(self):
extra_pref = "# output for interactive command: echo 'n: '; read n; seq 1 $n\n\n"
self.assertEqual(run_cmd_log_txt, extra_pref + "n: \n1\n2\n3\n4\n5\n")
+ def test_run_shell_cmd_qa_log(self):
+ """Test temporary log file for run_shell_cmd with qa_patterns"""
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd("echo 'n: '; read n; seq 1 $n", qa_patterns=[('n:', '5')])
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, "n: \n1\n2\n3\n4\n5\n")
+
+ run_cmd_logs = glob.glob(os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output', 'echo-*', 'out.txt'))
+ self.assertEqual(len(run_cmd_logs), 1)
+ run_cmd_log_txt = read_file(run_cmd_logs[0])
+ self.assertEqual(run_cmd_log_txt, "n: \n1\n2\n3\n4\n5\n")
+
def test_run_cmd_qa_trace(self):
"""Test run_cmd under --trace"""
- # replace log.experimental with log.warning to allow experimental code
- easybuild.tools.utilities._log.experimental = easybuild.tools.utilities._log.warning
- init_config(build_options={'trace': True})
+ # use of run_cmd/run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+ # --trace is enabled by default
self.mock_stdout(True)
self.mock_stderr(True)
(out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'})
@@ -830,7 +1221,7 @@ def test_run_cmd_qa_trace(self):
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
- self.assertEqual(stderr, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
pattern = r"^ >> running interactive command:\n"
pattern += r"\t\[started at: .*\]\n"
pattern += r"\t\[working dir: .*\]\n"
@@ -848,10 +1239,45 @@ def test_run_cmd_qa_trace(self):
self.mock_stdout(False)
self.mock_stderr(False)
self.assertEqual(stdout, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
+
+ def test_run_shell_cmd_qa_trace(self):
+ """Test run_shell_cmd with qa_patterns under --trace"""
+
+ # --trace is enabled by default
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ run_shell_cmd("echo 'n: '; read n; seq 1 $n", qa_patterns=[('n: ', '5')])
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ self.assertEqual(stderr, '')
+ pattern = r"^ >> running interactive shell command:\n"
+ pattern += r"\techo \'n: \'; read n; seq 1 \$n\n"
+ pattern += r"\t\[started at: .*\]\n"
+ pattern += r"\t\[working dir: .*\]\n"
+ pattern += r"\t\[output and state saved to .*\]\n"
+ pattern += r' >> command completed: exit 0, ran in .*'
+ self.assertTrue(re.search(pattern, stdout), "Pattern '%s' found in: %s" % (pattern, stdout))
+
+ # trace output can be disabled on a per-command basis
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ run_shell_cmd("echo 'n: '; read n; seq 1 $n", qa_patterns=[('n: ', '5')], hidden=True)
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+ self.assertEqual(stdout, '')
self.assertEqual(stderr, '')
def test_run_cmd_qa_answers(self):
"""Test providing list of answers in run_cmd_qa."""
+
+ # use of run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
cmd = "echo question; read x; echo $x; " * 2
qa = {"question": ["answer1", "answer2"]}
@@ -875,14 +1301,44 @@ def test_run_cmd_qa_answers(self):
self.assertEqual(out, "question\nanswer1\nquestion\nanswer2\n" * 2)
self.assertEqual(ec, 0)
+ def test_run_shell_cmd_qa_answers(self):
+ """Test providing list of answers for a question in run_shell_cmd."""
+
+ cmd = "echo question; read x; echo $x; " * 2
+ qa = [("question", ["answer1", "answer2"])]
+
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+ self.assertEqual(res.output, "question\nanswer1\nquestion\nanswer2\n")
+ self.assertEqual(res.exit_code, 0)
+
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, "Unknown type of answers encountered", run_shell_cmd, cmd,
+ qa_patterns=[('question', 1)])
+
+ # test cycling of answers
+ cmd = cmd * 2
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, qa_patterns=qa)
+ self.assertEqual(res.output, "question\nanswer1\nquestion\nanswer2\n" * 2)
+ self.assertEqual(res.exit_code, 0)
+
def test_run_cmd_simple(self):
"""Test return value for run_cmd in 'simple' mode."""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
with self.mocked_stdout_stderr():
self.assertEqual(True, run_cmd("echo hello", simple=True))
self.assertEqual(False, run_cmd("exit 1", simple=True, log_all=False, log_ok=False))
def test_run_cmd_cache(self):
"""Test caching for run_cmd"""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
with self.mocked_stdout_stderr():
(first_out, ec) = run_cmd("ulimit -u")
self.assertEqual(ec, 0)
@@ -932,7 +1388,7 @@ def test_run_shell_cmd_cache(self):
with self.mocked_stdout_stderr():
cached_res = RunShellCmdResult(cmd=cmd, output="123456", exit_code=123, stderr=None,
work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None,
- thread_id=None, task_id=None)
+ cmd_sh='/tmp/cmd.sh', thread_id=None, task_id=None)
run_shell_cmd.update_cache({(cmd, None): cached_res})
res = run_shell_cmd(cmd)
self.assertEqual(res.cmd, cmd)
@@ -952,7 +1408,7 @@ def test_run_shell_cmd_cache(self):
with self.mocked_stdout_stderr():
cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None,
work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None,
- thread_id=None, task_id=None)
+ cmd_sh='/tmp/cmd.sh', thread_id=None, task_id=None)
run_shell_cmd.update_cache({(cmd, 'foo'): cached_res})
res = run_shell_cmd(cmd, stdin='foo')
self.assertEqual(res.cmd, cmd)
@@ -965,11 +1421,20 @@ def test_run_shell_cmd_cache(self):
def test_parse_log_error(self):
"""Test basic parse_log_for_error functionality."""
- errors = parse_log_for_error("error failed", True)
+
+ # use of parse_log_for_error is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
+ with self.mocked_stdout_stderr():
+ errors = parse_log_for_error("error failed", True)
self.assertEqual(len(errors), 1)
def test_run_cmd_dry_run(self):
"""Test use of run_cmd function under (extended) dry run."""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
build_options = {
'extended_dry_run': True,
'silent': False,
@@ -978,35 +1443,38 @@ def test_run_cmd_dry_run(self):
cmd = "somecommand foo 123 bar"
- self.mock_stdout(True)
- run_cmd(cmd)
- stdout = self.get_stdout()
- self.mock_stdout(False)
+ with self.mocked_stdout_stderr():
+ run_cmd(cmd)
+ stdout = self.get_stdout()
expected = """ running command "somecommand foo 123 bar"\n"""
self.assertIn(expected, stdout)
# check disabling 'verbose'
- self.mock_stdout(True)
- run_cmd("somecommand foo 123 bar", verbose=False)
- stdout = self.get_stdout()
- self.mock_stdout(False)
+ with self.mocked_stdout_stderr():
+ run_cmd("somecommand foo 123 bar", verbose=False)
+ stdout = self.get_stdout()
self.assertNotIn(expected, stdout)
# check forced run_cmd
outfile = os.path.join(self.test_prefix, 'cmd.out')
self.assertNotExists(outfile)
- self.mock_stdout(True)
- run_cmd("echo 'This is always echoed' > %s" % outfile, force_in_dry_run=True)
- self.mock_stdout(False)
+ with self.mocked_stdout_stderr():
+ run_cmd("echo 'This is always echoed' > %s" % outfile, force_in_dry_run=True)
self.assertExists(outfile)
self.assertEqual(read_file(outfile), "This is always echoed\n")
# Q&A commands
- self.mock_stdout(True)
- run_cmd_qa("some_qa_cmd", {'question1': 'answer1'})
- stdout = self.get_stdout()
- self.mock_stdout(False)
+ with self.mocked_stdout_stderr():
+ run_shell_cmd("some_qa_cmd", qa_patterns=[('question1', 'answer1')])
+ stdout = self.get_stdout()
+
+ expected = """ running interactive shell command "some_qa_cmd"\n"""
+ self.assertIn(expected, stdout)
+
+ with self.mocked_stdout_stderr():
+ run_cmd_qa("some_qa_cmd", {'question1': 'answer1'})
+ stdout = self.get_stdout()
expected = """ running interactive command "some_qa_cmd"\n"""
self.assertIn(expected, stdout)
@@ -1062,6 +1530,10 @@ def test_run_shell_cmd_dry_run(self):
def test_run_cmd_list(self):
"""Test run_cmd with command specified as a list rather than a string"""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
cmd = ['/bin/sh', '-c', "echo hello"]
with self.mocked_stdout_stderr():
self.assertErrorRegex(EasyBuildError, "When passing cmd as a list then `shell` must be set explictely!",
@@ -1073,6 +1545,10 @@ def test_run_cmd_list(self):
def test_run_cmd_script(self):
"""Testing use of run_cmd with shell=False to call external scripts"""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
py_test_script = os.path.join(self.test_prefix, 'test.py')
write_file(py_test_script, '\n'.join([
'#!%s' % sys.executable,
@@ -1111,6 +1587,10 @@ def test_run_shell_cmd_no_bash(self):
def test_run_cmd_stream(self):
"""Test use of run_cmd with streaming output."""
+
+ # use of run_cmd is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
self.mock_stdout(True)
self.mock_stderr(True)
(out, ec) = run_cmd("echo hello", stream_output=True)
@@ -1122,7 +1602,7 @@ def test_run_cmd_stream(self):
self.assertEqual(ec, 0)
self.assertEqual(out, "hello\n")
- self.assertEqual(stderr, '')
+ self.assertTrue(stderr.strip().startswith("WARNING: Deprecated functionality"))
expected = [
"== (streaming) output for command 'echo hello':",
"hello",
@@ -1165,9 +1645,31 @@ def test_run_shell_cmd_stream(self):
for line in expected:
self.assertIn(line, stdout)
+ def test_run_shell_cmd_eof_stdin(self):
+ """Test use of run_shell_cmd with streaming output and blocking stdin read."""
+ cmd = 'timeout 1 cat -'
+
+ inp = 'hello\nworld\n'
+ # test with streaming output
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, stream_output=True, stdin=inp, fail_on_error=False)
+
+ self.assertEqual(res.exit_code, 0, "Streaming output: Command timed out")
+ self.assertEqual(res.output, inp)
+
+ # test with non-streaming output (proc.communicate() is used)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd, stdin=inp, fail_on_error=False)
+
+ self.assertEqual(res.exit_code, 0, "Non-streaming output: Command timed out")
+ self.assertEqual(res.output, inp)
+
def test_run_cmd_async(self):
"""Test asynchronously running of a shell command via run_cmd + complete_cmd."""
+ # use of run_cmd/check_async_cmd/get_output_from_process is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
os.environ['TEST'] = 'test123'
test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST"
@@ -1199,13 +1701,15 @@ def test_run_cmd_async(self):
# first check, only read first 12 output characters
# (otherwise we'll be waiting until command is completed)
- res = check_async_cmd(*cmd_info, output_read_size=12)
+ with self.mocked_stdout_stderr():
+ res = check_async_cmd(*cmd_info, output_read_size=12)
self.assertEqual(res, {'done': False, 'exit_code': None, 'output': 'sleeping...\n'})
# 2nd check with default output size (1024) gets full output
# (keep checking until command is fully done)
- while not res['done']:
- res = check_async_cmd(*cmd_info, output=res['output'])
+ with self.mocked_stdout_stderr():
+ while not res['done']:
+ res = check_async_cmd(*cmd_info, output=res['output'])
self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'})
# check asynchronous running of failing command
@@ -1214,14 +1718,16 @@ def test_run_cmd_async(self):
cmd_info = run_cmd(error_test_cmd, asynchronous=True)
time.sleep(1)
error_pattern = 'cmd ".*" exited with exit code 123'
- self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info)
with self.mocked_stdout_stderr():
cmd_info = run_cmd(error_test_cmd, asynchronous=True)
- res = check_async_cmd(*cmd_info, fail_on_error=False)
+ res = check_async_cmd(*cmd_info, fail_on_error=False)
# keep checking until command is fully done
- while not res['done']:
- res = check_async_cmd(*cmd_info, fail_on_error=False, output=res['output'])
+ with self.mocked_stdout_stderr():
+ while not res['done']:
+ res = check_async_cmd(*cmd_info, fail_on_error=False, output=res['output'])
self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"})
# also test with a command that produces a lot of output,
@@ -1244,10 +1750,11 @@ def test_run_cmd_async(self):
ec = proc.poll()
self.assertEqual(ec, None)
- while ec is None:
- time.sleep(1)
- output += get_output_from_process(proc)
- ec = proc.poll()
+ with self.mocked_stdout_stderr():
+ while ec is None:
+ time.sleep(1)
+ output += get_output_from_process(proc)
+ ec = proc.poll()
with self.mocked_stdout_stderr():
out, ec = complete_cmd(*cmd_info, simple=False, output=output)
@@ -1260,8 +1767,9 @@ def test_run_cmd_async(self):
cmd_info = run_cmd(verbose_test_cmd, asynchronous=True)
error_pattern = r"Number of output bytes to read should be a positive integer value \(or zero\)"
- self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size=-1)
- self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size='foo')
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size=-1)
+ self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size='foo')
# with output_read_size set to 0, no output is read yet, only status of command is checked
with self.mocked_stdout_stderr():
@@ -1277,8 +1785,9 @@ def test_run_cmd_async(self):
self.assertTrue(res['output'].startswith('start\n'))
self.assertFalse(res['output'].endswith('\ndone\n'))
# keep checking until command is complete
- while not res['done']:
- res = check_async_cmd(*cmd_info, output=res['output'])
+ with self.mocked_stdout_stderr():
+ while not res['done']:
+ res = check_async_cmd(*cmd_info, output=res['output'])
self.assertEqual(res['done'], True)
self.assertEqual(res['exit_code'], 0)
self.assertEqual(len(res['output']), 435661)
@@ -1345,13 +1854,19 @@ def test_run_shell_cmd_async(self):
self.assertTrue(res.output.endswith('\nfoo501000\ndone\n'))
def test_check_log_for_errors(self):
+ """Test for check_log_for_errors"""
+
+ # use of check_log_for_errors is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
os.close(fd)
- self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [42])
- self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [(42, IGNORE)])
- self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", "invalid-mode")])
- self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", IGNORE, "")])
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [42])
+ self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [(42, IGNORE)])
+ self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", "invalid-mode")])
+ self.assertErrorRegex(EasyBuildError, "Invalid input:", check_log_for_errors, "", [("42", IGNORE, "")])
input_text = "\n".join([
"OK",
@@ -1366,20 +1881,24 @@ def test_check_log_for_errors(self):
r"\tthe process crashed with 0"
# String promoted to list
- self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
- r"\b(error|crashed)\b")
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
+ r"\b(error|crashed)\b")
# List of string(s)
- self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
- [r"\b(error|crashed)\b"])
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
+ [r"\b(error|crashed)\b"])
# List of tuple(s)
- self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
- [(r"\b(error|crashed)\b", ERROR)])
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text,
+ [(r"\b(error|crashed)\b", ERROR)])
expected_msg = "Found 2 potential error(s) in command output:\n"\
"\terror found\n"\
"\tthe process crashed with 0"
init_logging(logfile, silent=True)
- check_log_for_errors(input_text, [(r"\b(error|crashed)\b", WARN)])
+ with self.mocked_stdout_stderr():
+ check_log_for_errors(input_text, [(r"\b(error|crashed)\b", WARN)])
stop_logging(logfile)
self.assertIn(expected_msg, read_file(logfile))
@@ -1388,12 +1907,13 @@ def test_check_log_for_errors(self):
r"\ttest failed"
write_file(logfile, '')
init_logging(logfile, silent=True)
- self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [
- r"\berror\b",
- (r"\ballowed-test failed\b", IGNORE),
- (r"(?i)\bCRASHED\b", WARN),
- "fail"
- ])
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [
+ r"\berror\b",
+ (r"\ballowed-test failed\b", IGNORE),
+ (r"(?i)\bCRASHED\b", WARN),
+ "fail"
+ ])
stop_logging(logfile)
expected_msg = "Found 1 potential error(s) in command output:\n\tthe process crashed with 0"
self.assertIn(expected_msg, read_file(logfile))
@@ -1402,6 +1922,10 @@ def test_run_cmd_with_hooks(self):
"""
Test running command with run_cmd with pre/post run_shell_cmd hooks in place.
"""
+
+ # use of run_cmd/run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
cwd = os.getcwd()
hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
@@ -1444,6 +1968,17 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
])
self.assertEqual(stdout, expected_stdout)
+ with self.mocked_stdout_stderr():
+ run_shell_cmd("sleep 2; make", qa_patterns=[('q', 'a')])
+ stdout = self.get_stdout()
+
+ expected_stdout = '\n'.join([
+ "pre-run hook interactive 'sleep 2; make' in %s" % cwd,
+ "post-run hook interactive 'sleep 2; echo make' (exit code: 0, output: 'make\n')",
+ '',
+ ])
+ self.assertEqual(stdout, expected_stdout)
+
with self.mocked_stdout_stderr():
run_cmd_qa("sleep 2; make", qa={})
stdout = self.get_stdout()
@@ -1497,12 +2032,136 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
stdout = self.get_stdout()
expected_stdout = '\n'.join([
- "pre-run hook 'make' in %s" % cwd,
+ f"pre-run hook 'make' in {cwd}",
"post-run hook 'echo make' (exit code: 0, output: 'make\n')",
'',
])
self.assertEqual(stdout, expected_stdout)
+ # also check in dry run mode, to verify that pre-run_shell_cmd hook is triggered sufficiently early
+ update_build_option('extended_dry_run', True)
+
+ with self.mocked_stdout_stderr():
+ run_shell_cmd("make")
+ stdout = self.get_stdout()
+
+ expected_stdout = '\n'.join([
+ "pre-run hook 'make' in %s" % cwd,
+ ' running shell command "echo make"',
+ ' (in %s)' % cwd,
+ '',
+ ])
+ self.assertEqual(stdout, expected_stdout)
+
+ # also check with trace output enabled
+ update_build_option('extended_dry_run', False)
+ update_build_option('trace', True)
+
+ with self.mocked_stdout_stderr():
+ run_shell_cmd("make")
+ stdout = self.get_stdout()
+
+ regex = re.compile('>> running shell command:\n\techo make', re.M)
+ self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+
+ def test_run_shell_cmd_delete_cwd(self):
+ """
+ Test commands that destroy directories inside initial working directory
+ """
+ workdir = os.path.join(self.test_prefix, 'workdir')
+ sub_workdir = os.path.join(workdir, 'subworkdir')
+
+ # 1. test destruction of CWD which is a subdirectory inside original working directory
+ cmd_subworkdir_rm = (
+ "echo 'Command that jumps to subdir and removes it' && "
+ f"cd {sub_workdir} && pwd && rm -rf {sub_workdir} && "
+ "echo 'Working sub-directory removed.'"
+ )
+
+ # 1.a. in a robust system
+ expected_output = (
+ "Command that jumps to subdir and removes it\n"
+ f"{sub_workdir}\n"
+ "Working sub-directory removed.\n"
+ )
+
+ mkdir(sub_workdir, parents=True)
+ with self.mocked_stdout_stderr():
+ res = run_shell_cmd(cmd_subworkdir_rm, work_dir=workdir)
+
+ self.assertEqual(res.cmd, cmd_subworkdir_rm)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, expected_output)
+ self.assertEqual(res.stderr, None)
+ self.assertEqual(res.work_dir, workdir)
+
+ # 1.b. in a flaky system that ends up in an unknown CWD after execution
+ mkdir(sub_workdir, parents=True)
+ fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
+ os.close(fd)
+
+ with self.mocked_stdout_stderr():
+ with mock.patch('os.getcwd') as mock_getcwd:
+ mock_getcwd.side_effect = [
+ workdir,
+ FileNotFoundError(),
+ ]
+ init_logging(logfile, silent=True)
+ res = run_shell_cmd(cmd_subworkdir_rm, work_dir=workdir)
+ stop_logging(logfile)
+
+ self.assertEqual(res.cmd, cmd_subworkdir_rm)
+ self.assertEqual(res.exit_code, 0)
+ self.assertEqual(res.output, expected_output)
+ self.assertEqual(res.stderr, None)
+ self.assertEqual(res.work_dir, workdir)
+
+ expected_warning = f"Changing back to initial working directory: {workdir}\n"
+ logtxt = read_file(logfile)
+ self.assertTrue(logtxt.endswith(expected_warning))
+
+ # 2. test destruction of CWD which is main working directory passed to run_shell_cmd
+ cmd_workdir_rm = (
+ "echo 'Command that removes working directory' && pwd && "
+ f"rm -rf {workdir} && echo 'Working directory removed.'"
+ )
+
+ error_pattern = rf"Failed to return to .*/{os.path.basename(self.test_prefix)}/workdir after executing command"
+
+ mkdir(workdir, parents=True)
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd_workdir_rm, work_dir=workdir)
+
+ def test_run_cmd_sysroot(self):
+ """Test with_sysroot option of run_cmd function."""
+
+ # use of run_cmd/run_cmd_qa is deprecated, so we need to allow it here
+ self.allow_deprecated_behaviour()
+
+ # put fake /bin/bash in place that will be picked up when using run_cmd with with_sysroot=True
+ bin_bash = os.path.join(self.test_prefix, 'bin', 'bash')
+ bin_bash_txt = '\n'.join([
+ "#!/bin/bash",
+ "echo 'Hi there I am a fake /bin/bash in %s'" % self.test_prefix,
+ '/bin/bash "$@"',
+ ])
+ write_file(bin_bash, bin_bash_txt)
+ adjust_permissions(bin_bash, stat.S_IXUSR)
+
+ update_build_option('sysroot', self.test_prefix)
+
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd("echo hello")
+ self.assertEqual(ec, 0)
+ self.assertTrue(out.startswith("Hi there I am a fake /bin/bash in"))
+ self.assertTrue(out.endswith("\nhello\n"))
+
+ # picking up on alternate sysroot is enabled by default, but can be disabled via with_sysroot=False
+ with self.mocked_stdout_stderr():
+ (out, ec) = run_cmd("echo hello", with_sysroot=False)
+ self.assertEqual(ec, 0)
+ self.assertEqual(out, "hello\n")
+
def suite():
""" returns all the testcases in this module """
diff --git a/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py b/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py
index e27c0c66d0..7d2ef2d6da 100644
--- a/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py
+++ b/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2020 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/f/fftw.py b/test/framework/sandbox/easybuild/easyblocks/f/fftw.py
index 194cead197..0871ce62e1 100644
--- a/test/framework/sandbox/easybuild/easyblocks/f/fftw.py
+++ b/test/framework/sandbox/easybuild/easyblocks/f/fftw.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foo.py b/test/framework/sandbox/easybuild/easyblocks/f/foo.py
index 07e21608fa..b5a91d9503 100644
--- a/test/framework/sandbox/easybuild/easyblocks/f/foo.py
+++ b/test/framework/sandbox/easybuild/easyblocks/f/foo.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py
index 3d64b1cf97..6415d6e6e1 100644
--- a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py
+++ b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py
index a85b2482fa..5ee5aae1c6 100644
--- a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py
+++ b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py
index d2faf66053..8ba475f700 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py
new file mode 100644
index 0000000000..a7fa78d986
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py
@@ -0,0 +1,35 @@
+##
+# Copyright 2009-2024 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Test EasyBlocks building and installing dummy extensions with customized methods
+
+@author: Alex Domingo (Vrije Universiteit Brussel)
+"""
+
+from easybuild.easyblocks.generic.customdummyextension import CustomDummyExtension
+
+
+class ChildCustomDummyExtension(CustomDummyExtension):
+ """Extension EasyBlock inheriting customized install step"""
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py
new file mode 100644
index 0000000000..e72b97940a
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py
@@ -0,0 +1,35 @@
+##
+# Copyright 2009-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Test EasyBlocks building and installing dummy extensions with deprecated methods
+
+@author: Alex Domingo (Vrije Universiteit Brussel)
+"""
+
+from easybuild.easyblocks.generic.deprecateddummyextension import DeprecatedDummyExtension
+
+
+class ChildDeprecatedDummyExtension(DeprecatedDummyExtension):
+ """Extension EasyBlock inheriting deprecated install step"""
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py
index 4ad690fe5c..b82668434c 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py
new file mode 100644
index 0000000000..17a4434fe0
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py
@@ -0,0 +1,47 @@
+##
+# Copyright 2009-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Test EasyBlocks building and installing dummy extensions with customized methods
+
+@author: Alex Domingo (Vrije Universiteit Brussel)
+"""
+
+from easybuild.easyblocks.generic.dummyextension import DummyExtension
+
+
+class CustomDummyExtension(DummyExtension):
+ """Extension EasyBlock with customized install step"""
+
+ def pre_install_extension(self):
+
+ return "Extension installed with custom pre_install_extension()"
+
+ def install_extension(self):
+
+ return "Extension installed with custom install_extension()"
+
+ def post_install_extension(self):
+
+ return "Extension installed with custom post_install_extension()"
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py
new file mode 100644
index 0000000000..eca1ee1435
--- /dev/null
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py
@@ -0,0 +1,47 @@
+##
+# Copyright 2009-2024 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Test EasyBlocks building and installing dummy extensions with deprecated methods
+
+@author: Alex Domingo (Vrije Universiteit Brussel)
+"""
+
+from easybuild.easyblocks.generic.dummyextension import DummyExtension
+
+
+class DeprecatedDummyExtension(DummyExtension):
+ """Extension EasyBlock with deprecated install step"""
+
+ def prerun(self):
+
+ return "Extension installed with custom prerun()"
+
+ def run(self):
+
+ return "Extension installed with custom run()"
+
+ def postrun(self):
+
+ return "Extension installed with custom postrun()"
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py
index d754117b39..3466ce28f2 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py b/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py
index 6b258c87d6..dd89704638 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2020 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py
index 33ab7db67f..72252a46f4 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py
index 0321602f3f..9c01951adf 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2020 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py
index fb81a51c96..5151221a13 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py
index 28792ff4df..37818b4954 100644
--- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -60,7 +60,17 @@ def required_deps(self):
else:
raise EasyBuildError("Dependencies for %s are unknown!", self.name)
- def run(self, *args, **kwargs):
+ def pre_install_extension(self):
+ """
+ Prepare installation of toy extension.
+ """
+ super(Toy_Extension, self).pre_install_extension()
+
+ if self.src:
+ super(Toy_Extension, self).install_extension(unpack_src=True)
+ EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg)
+
+ def install_extension(self, *args, **kwargs):
"""
Install toy extension.
"""
@@ -72,17 +82,7 @@ def run(self, *args, **kwargs):
return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper().replace('-', '_'), self.name)
- def prerun(self):
- """
- Prepare installation of toy extension.
- """
- super(Toy_Extension, self).prerun()
-
- if self.src:
- super(Toy_Extension, self).run(unpack_src=True)
- EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg)
-
- def run_async(self, thread_pool):
+ def install_extension_async(self, thread_pool):
"""
Install toy extension asynchronously.
"""
@@ -95,11 +95,12 @@ def run_async(self, thread_pool):
return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(),
fail_on_error=False, task_id=task_id, work_dir=os.getcwd())
- def postrun(self):
+ def post_install_extension(self):
"""
Wrap up installation of toy extension.
"""
- super(Toy_Extension, self).postrun()
+ super(Toy_Extension, self).post_install_extension()
+
EB_toy.install_step(self.master, name=self.name)
def sanity_check_step(self, *args, **kwargs):
diff --git a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py
index 89fba75c30..db929d4d31 100644
--- a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py
+++ b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py
index 2c0ba5a9e5..20cc2f4b4a 100644
--- a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py
+++ b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py
@@ -1,5 +1,5 @@
##
-# Copyright 2021-2023 Ghent University
+# Copyright 2021-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openblas.py b/test/framework/sandbox/easybuild/easyblocks/o/openblas.py
index bfb118cefa..e5dc093347 100644
--- a/test/framework/sandbox/easybuild/easyblocks/o/openblas.py
+++ b/test/framework/sandbox/easybuild/easyblocks/o/openblas.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py b/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py
index 630d5413a7..b110b4f921 100644
--- a/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py
+++ b/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py
index d404f81648..3ded1247f4 100644
--- a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py
+++ b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py
index 19b19d32dd..5727dcd0e0 100644
--- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py
+++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -150,20 +150,20 @@ def required_deps(self):
else:
raise EasyBuildError("Dependencies for %s are unknown!", self.name)
- def prerun(self):
+ def pre_install_extension(self):
"""
Prepare installation of toy as extension.
"""
- super(EB_toy, self).run(unpack_src=True)
+ super(EB_toy, self).install_extension(unpack_src=True)
self.configure_step()
- def run(self):
+ def install_extension(self):
"""
Install toy as extension.
"""
self.build_step()
- def run_async(self, thread_pool):
+ def install_extension_async(self, thread_pool):
"""
Asynchronous installation of toy as extension.
"""
@@ -172,7 +172,7 @@ def run_async(self, thread_pool):
return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(),
fail_on_error=False, task_id=task_id, work_dir=os.getcwd())
- def postrun(self):
+ def post_install_extension(self):
"""
Wrap up installation of toy as extension.
"""
diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py
index 43d7433265..7395c99247 100644
--- a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py
+++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py
index 24f8321a09..8673dd8f60 100644
--- a/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py
+++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py
@@ -1,5 +1,5 @@
##
-# Copyright 2020-2023 Ghent University
+# Copyright 2020-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py
index a1fc62c69b..a3ef746d53 100644
--- a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py
+++ b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/tools/__init__.py b/test/framework/sandbox/easybuild/tools/__init__.py
index 1fbb2401b7..389c5ff367 100644
--- a/test/framework/sandbox/easybuild/tools/__init__.py
+++ b/test/framework/sandbox/easybuild/tools/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py
index 3a9383c79f..a519340298 100644
--- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py
+++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py
@@ -1,5 +1,5 @@
##
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py
index 4d33d3d1e7..9113d89045 100644
--- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py
+++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py
index 5d8dec26c8..424fb55d43 100644
--- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py
+++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py
index 3cbb5cb60d..a0e0429bfe 100644
--- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py
+++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/style.py b/test/framework/style.py
index 298b165c24..0c3060b4cb 100644
--- a/test/framework/style.py
+++ b/test/framework/style.py
@@ -1,5 +1,5 @@
##
-# Copyright 2016-2023 Ghent University
+# Copyright 2016-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,10 +40,7 @@
try:
import pycodestyle # noqa
except ImportError:
- try:
- import pep8 # noqa
- except ImportError:
- pass
+ pass
class StyleTest(EnhancedTestCase):
@@ -51,8 +48,8 @@ class StyleTest(EnhancedTestCase):
def test_style_conformance(self):
"""Check the easyconfigs for style"""
- if not ('pycodestyle' in sys.modules or 'pep8' in sys.modules):
- print("Skipping style checks (no pycodestyle or pep8 available)")
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_style_conformance pycodestyle is not available")
return
# all available easyconfig files
@@ -66,8 +63,8 @@ def test_style_conformance(self):
def test_check_trailing_whitespace(self):
"""Test for trailing whitespace check."""
- if not ('pycodestyle' in sys.modules or 'pep8' in sys.modules):
- print("Skipping trailing whitespace checks (no pycodestyle or pep8 available)")
+ if 'pycodestyle' not in sys.modules:
+ print("Skipping test_check_trailing_whitespace is not available")
return
lines = [
diff --git a/test/framework/suite.py b/test/framework/suite.py
index e109848340..97522ffacf 100755
--- a/test/framework/suite.py
+++ b/test/framework/suite.py
@@ -1,6 +1,6 @@
#!/usr/bin/python
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py
index 6d8395a5fc..103116c4c6 100644
--- a/test/framework/systemtools.py
+++ b/test/framework/systemtools.py
@@ -1,5 +1,5 @@
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -341,7 +341,7 @@ def mocked_run_shell_cmd(cmd, **kwargs):
}
if cmd in known_cmds:
return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd(),
- out_file=None, err_file=None, thread_id=None, task_id=None)
+ out_file=None, err_file=None, cmd_sh=None, thread_id=None, task_id=None)
else:
return run_shell_cmd(cmd, **kwargs)
@@ -774,7 +774,7 @@ def test_gcc_version_darwin(self):
out = "Apple LLVM version 7.0.0 (clang-700.1.76)"
cwd = os.getcwd()
mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd,
- out_file=None, err_file=None, thread_id=None, task_id=None)
+ out_file=None, err_file=None, cmd_sh=None, thread_id=None, task_id=None)
st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res
self.assertEqual(get_gcc_version(), None)
@@ -1033,15 +1033,15 @@ def test_check_linked_shared_libs(self):
self.assertEqual(check_linked_shared_libs(txt_path), None)
self.assertEqual(check_linked_shared_libs(broken_symlink_path), None)
- bin_ls_path = which('ls')
+ bin_bash_path = which('bash')
os_type = get_os_type()
if os_type == LINUX:
with self.mocked_stdout_stderr():
- res = run_shell_cmd("ldd %s" % bin_ls_path)
+ res = run_shell_cmd("ldd %s" % bin_bash_path)
elif os_type == DARWIN:
with self.mocked_stdout_stderr():
- res = run_shell_cmd("otool -L %s" % bin_ls_path)
+ res = run_shell_cmd("otool -L %s" % bin_bash_path)
else:
raise EasyBuildError("Unknown OS type: %s" % os_type)
@@ -1061,7 +1061,7 @@ def test_check_linked_shared_libs(self):
self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None)
self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None)
self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None)
- for path in (bin_ls_path, lib_path):
+ for path in (bin_bash_path, lib_path):
# path may not exist, especially for library paths obtained via 'otool -L' on macOS
if os.path.exists(path):
error_msg = "Check on linked libs should pass for %s with %s" % (path, pattern_named_args)
@@ -1078,7 +1078,7 @@ def test_check_linked_shared_libs(self):
self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None)
self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None)
self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None)
- for path in (bin_ls_path, lib_path):
+ for path in (bin_bash_path, lib_path):
error_msg = "Check on linked libs should fail for %s with %s" % (path, pattern_named_args)
self.assertFalse(check_linked_shared_libs(path, **pattern_named_args), error_msg)
diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
index 918b344dcf..2e8c7893b1 100644
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -50,6 +50,7 @@
from easybuild.tools.environment import setvar
from easybuild.tools.filetools import adjust_permissions, copy_dir, find_eb_script, mkdir
from easybuild.tools.filetools import read_file, symlink, write_file, which
+from easybuild.tools.modules import EnvironmentModules
from easybuild.tools.run import run_shell_cmd
from easybuild.tools.systemtools import get_shared_lib_ext
from easybuild.tools.toolchain.mpi import get_mpi_cmd_template
@@ -91,7 +92,7 @@ def get_toolchain(self, name, version=None):
def test_toolchain(self):
"""Test whether toolchain is initialized correctly."""
test_ecs = os.path.join('test', 'framework', 'easyconfigs', 'test_ecs')
- ec_file = find_full_path(os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4.eb'))
+ ec_file = find_full_path(os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4-GCC-4.9.3-2.26.eb'))
ec = EasyConfig(ec_file, validate=False)
tc = ec.toolchain
self.assertIn('debug', tc.options)
@@ -144,7 +145,7 @@ def test_is_system_toolchain(self):
def test_toolchain_prepare_sysroot(self):
"""Test build environment setup done by Toolchain.prepare in case --sysroot is specified."""
- sysroot = os.path.join(self.test_prefix, 'test', 'alternate', 'sysroot')
+ sysroot = os.path.join(self.test_prefix, 'test', 'alternative', 'sysroot')
sysroot_pkgconfig = os.path.join(sysroot, 'usr', 'lib', 'pkgconfig')
mkdir(sysroot_pkgconfig, parents=True)
init_config(build_options={'sysroot': sysroot})
@@ -152,8 +153,7 @@ def test_toolchain_prepare_sysroot(self):
# clean environment
self.unset_compiler_env_vars()
- if 'PKG_CONFIG_PATH' in os.environ:
- del os.environ['PKG_CONFIG_PATH']
+ os.environ.pop('PKG_CONFIG_PATH', None)
self.assertEqual(os.getenv('PKG_CONFIG_PATH'), None)
@@ -203,8 +203,7 @@ def unset_compiler_env_vars(self):
env_vars.extend(['OMPI_%s' % x for x in comp_env_vars])
for key in env_vars:
- if key in os.environ:
- del os.environ[key]
+ os.environ.pop(key, None)
def test_toolchain_compiler_env_vars(self):
"""Test whether environment variables for compilers are defined by toolchain mechanism."""
@@ -322,8 +321,7 @@ def test_toolchain_compiler_env_vars(self):
init_config(build_options={'minimal_build_env': 'CC:gcc,CXX:g++,CFLAGS:-O2,CXXFLAGS:-O3 -g,FC:gfortan'})
for key in ['CFLAGS', 'CXXFLAGS', 'FC']:
- if key in os.environ:
- del os.environ[key]
+ os.environ.pop(key, None)
self.mock_stderr(True)
self.mock_stdout(True)
@@ -473,6 +471,10 @@ def test_cray_reset(self):
init_config(build_options={'optarch': 'test', 'silent': True})
for tcname in ['CrayGNU', 'CrayCCE', 'CrayIntel']:
+ # Cray* modules do not unload other Cray* modules thus loading a second Cray* module
+ # makes environment inconsistent which is not allowed by Environment Modules tool
+ if isinstance(self.modtool, EnvironmentModules):
+ self.modtool.purge()
tc = self.get_toolchain(tcname, version='2015.06-XC')
tc.set_options({'dynamic': True})
with self.mocked_stdout_stderr():
@@ -1237,7 +1239,7 @@ def test_fft_env_vars_intel(self):
self.assertEqual(tc.get_variable('LIBFFT'), libfft)
self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt)
- fft_lib_dir = os.path.join(modules.get_software_root('imkl'), 'mkl/2021.4.0/lib/intel64')
+ fft_lib_dir = os.path.join(modules.get_software_root('imkl'), 'mkl/2021.4/lib/intel64')
self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir)
tc = self.get_toolchain('intel', version='2021b')
@@ -1359,8 +1361,11 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'):
])
write_file(imkl_fftw_module_path, imkl_fftw_mod_txt)
- subdir = 'mkl/%s/lib/intel64' % imklver
+ # put "latest" symbolic link to short version, used in newer MKL
+ imklshortver = '.'.join(imklver.split('.')[:2])
+ subdir = 'mkl/%s/lib/intel64' % imklshortver
os.makedirs(os.path.join(imkl_dir, subdir))
+ os.symlink(imklshortver, os.path.join(imkl_dir, 'mkl', 'latest'))
for fftlib in mkl_libs:
write_file(os.path.join(imkl_dir, subdir, 'lib%s.a' % fftlib), 'foo')
subdir = 'lib'
@@ -1544,6 +1549,37 @@ def test_intel_toolchain_oneapi(self):
self.assertEqual(os.getenv('F90'), 'ifx')
self.assertEqual(os.getenv('FC'), 'ifx')
+ self.modtool.purge()
+ tc = self.get_toolchain('intel-compilers', version='2024.0.0')
+ tc.prepare()
+
+ # by default (for version >= 2024.0.0): oneAPI C/C++ compiler + oneAPI Fortran compiler
+ self.assertEqual(os.getenv('CC'), 'icx')
+ self.assertEqual(os.getenv('CXX'), 'icpx')
+ self.assertEqual(os.getenv('F77'), 'ifx')
+ self.assertEqual(os.getenv('F90'), 'ifx')
+ self.assertEqual(os.getenv('FC'), 'ifx')
+
+ self.modtool.purge()
+ tc = self.get_toolchain('intel-compilers', version='2024.0.0')
+ tc.set_options({'oneapi_fortran': False})
+ tc.prepare()
+ self.assertEqual(os.getenv('CC'), 'icx')
+ self.assertEqual(os.getenv('CXX'), 'icpx')
+ self.assertEqual(os.getenv('F77'), 'ifort')
+ self.assertEqual(os.getenv('F90'), 'ifort')
+ self.assertEqual(os.getenv('FC'), 'ifort')
+
+ self.modtool.purge()
+ tc = self.get_toolchain('intel-compilers', version='2024.0.0')
+ tc.set_options({'oneapi_c_cxx': False, 'oneapi_fortran': False})
+ tc.prepare()
+ self.assertEqual(os.getenv('CC'), 'icc')
+ self.assertEqual(os.getenv('CXX'), 'icpc')
+ self.assertEqual(os.getenv('F77'), 'ifort')
+ self.assertEqual(os.getenv('F90'), 'ifort')
+ self.assertEqual(os.getenv('FC'), 'ifort')
+
self.modtool.purge()
tc = self.get_toolchain('intel', version='2021b')
tc.set_options({'oneapi_c_cxx': True})
@@ -2177,6 +2213,10 @@ def test_independence(self):
# purposely obtain toolchains several times in a row, value for $CFLAGS should not change
for _ in range(3):
for tcname, tcversion in toolchains:
+ # Cray* modules do not unload other Cray* modules thus loading a second Cray* module
+ # makes environment inconsistent which is not allowed by Environment Modules tool
+ if isinstance(self.modtool, EnvironmentModules):
+ self.modtool.purge()
tc = get_toolchain({'name': tcname, 'version': tcversion}, {},
mns=ActiveMNS(), modtool=self.modtool)
# also check whether correct compiler flag for OpenMP is used while we're at it
@@ -2286,7 +2326,8 @@ def test_compiler_cache(self):
"#!/bin/bash",
"echo 'This is a %s wrapper'" % cache_tool,
"NAME=${0##*/}",
- "comm=$(which -a $NAME | sed 1d)",
+ "comms=($(which -a $NAME))",
+ "comm=${comms[1]}", # First entry is this wrapper, take 2nd
"$comm $@",
"exit 0"
]
@@ -2352,8 +2393,7 @@ def test_rpath_args_script(self):
"""Test rpath_args.py script"""
# $LIBRARY_PATH affects result of rpath_args.py, so make sure it's not set
- if 'LIBRARY_PATH' in os.environ:
- del os.environ['LIBRARY_PATH']
+ os.environ.pop('LIBRARY_PATH', None)
script = find_eb_script('rpath_args.py')
diff --git a/test/framework/toolchainvariables.py b/test/framework/toolchainvariables.py
index db90378d8b..e61bb9031c 100644
--- a/test/framework/toolchainvariables.py
+++ b/test/framework/toolchainvariables.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index 61a9cca2e5..69c47e768d 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
##
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -57,7 +57,7 @@
from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, move_file
from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file
from easybuild.tools.module_generator import ModuleGeneratorTcl
-from easybuild.tools.modules import Lmod
+from easybuild.tools.modules import EnvironmentModules, Lmod
from easybuild.tools.run import run_shell_cmd
from easybuild.tools.utilities import nub
from easybuild.tools.systemtools import get_shared_lib_ext
@@ -262,7 +262,7 @@ def test_toy_broken(self):
broken_toy_ec = os.path.join(tmpdir, "toy-broken.eb")
toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
broken_toy_ec_txt = read_file(toy_ec_file)
- broken_toy_ec_txt += "checksums = ['clearywrongMD5checksumoflength32']"
+ broken_toy_ec_txt += "checksums = ['clearywrongSHA256checksumoflength64-0123456789012345678901234567']"
write_file(broken_toy_ec, broken_toy_ec_txt)
error_regex = "Checksum verification .* failed"
self.assertErrorRegex(EasyBuildError, error_regex, self.run_test_toy_build_with_output, ec_file=broken_toy_ec,
@@ -366,8 +366,7 @@ def test_toy_tweaked(self):
expected += "oh hai!"
# setting $LMOD_QUIET results in suppression of printed message with Lmod & module files in Tcl syntax
- if 'LMOD_QUIET' in os.environ:
- del os.environ['LMOD_QUIET']
+ os.environ.pop('LMOD_QUIET', None)
self.modtool.use(os.path.join(self.test_installpath, 'modules', 'all'))
out = self.modtool.run_module('load', 'toy/0.0-tweaked', return_output=True)
@@ -775,14 +774,9 @@ def test_toy_group_check(self):
self.mock_stdout(False)
if get_module_syntax() == 'Tcl':
- pattern = "Can't generate robust check in TCL modules for users belonging to group %s." % group_name
- regex = re.compile(pattern, re.M)
- self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt))
-
- elif get_module_syntax() == 'Lua':
- lmod_version = os.getenv('LMOD_VERSION', 'NOT_FOUND')
- if LooseVersion(lmod_version) >= LooseVersion('6.0.8'):
- toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0.lua')
+ module_version = LooseVersion(self.modtool.version)
+ if isinstance(self.modtool, EnvironmentModules) and module_version >= LooseVersion('4.6.0'):
+ toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
toy_mod_txt = read_file(toy_mod)
if isinstance(group, tuple):
@@ -793,17 +787,35 @@ def test_toy_group_check(self):
error_msg_pattern = "You are not part of '%s' group of users" % group_name
pattern = '\n'.join([
- r'^if not \( userInGroup\("%s"\) \) then' % group_name,
- r' LmodError\("%s[^"]*"\)' % error_msg_pattern,
- r'end$',
+ r'^if \{ \!\[ module-info usergroups %s \] \} \{' % group_name,
+ r' error "%s[^"]*"' % error_msg_pattern,
+ r'\}$',
])
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_mod_txt))
else:
- pattern = r"Can't generate robust check in Lua modules for users belonging to group %s. "
- pattern += r"Lmod version not recent enough \(%s\), should be >= 6.0.8" % lmod_version
- regex = re.compile(pattern % group_name, re.M)
+ pattern = "Can't generate robust check in Tcl modules for users belonging to group %s." % group_name
+ regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt))
+
+ elif get_module_syntax() == 'Lua':
+ toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0.lua')
+ toy_mod_txt = read_file(toy_mod)
+
+ if isinstance(group, tuple):
+ group_name = group[0]
+ error_msg_pattern = "Hey, you're not in the '%s' group!" % group_name
+ else:
+ group_name = group
+ error_msg_pattern = "You are not part of '%s' group of users" % group_name
+
+ pattern = '\n'.join([
+ r'^if not \( userInGroup\("%s"\) \) then' % group_name,
+ r' LmodError\("%s[^"]*"\)' % error_msg_pattern,
+ r'end$',
+ ])
+ regex = re.compile(pattern, re.M)
+ self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, toy_mod_txt))
else:
self.fail("Unknown module syntax: %s" % get_module_syntax())
@@ -861,9 +873,9 @@ def test_toy_hierarchical(self):
# check that toolchain load is expanded to loads for toolchain dependencies,
# except for the ones that extend $MODULEPATH to make the toy module available
if get_module_syntax() == 'Tcl':
- load_regex_template = "load %s"
+ load_regex_template = "(load|depends-on) %s"
elif get_module_syntax() == 'Lua':
- load_regex_template = r'load\("%s/.*"\)'
+ load_regex_template = r'(load|depends_on)\("%s/.*"\)'
else:
self.fail("Unknown module syntax: %s" % get_module_syntax())
@@ -1444,7 +1456,7 @@ def test_toy_extension_extract_cmd(self):
])
write_file(test_ec, test_ec_txt)
- error_pattern = r"shell command 'unzip \.\.\.' failed in extensions step for test.eb"
+ error_pattern = r"shell command 'unzip \.\.\.' failed with exit code 9 in extensions step for test.eb"
with self.mocked_stdout_stderr():
# for now, we expect subprocess.CalledProcessError, but eventually 'run' function will
# do proper error reporting
@@ -1727,8 +1739,10 @@ def test_module_only(self):
# make sure load statements for dependencies are included in additional module file generated with --module-only
modtxt = read_file(toy_mod)
- self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module")
- self.assertTrue(re.search('load.*GCC/6.4.0-2.28', modtxt), "load statement for GCC/6.4.0-2.28 found in module")
+ self.assertTrue(re.search('(load|depends[-_]on).*intel/2018a', modtxt),
+ "load statement for intel/2018a found in module")
+ self.assertTrue(re.search('(load|depends[-_]on).*GCC/6.4.0-2.28', modtxt),
+ "load statement for GCC/6.4.0-2.28 found in module")
os.remove(toy_mod)
@@ -1770,7 +1784,8 @@ def test_module_only(self):
# make sure load statements for dependencies are included
modtxt = read_file(toy_core_mod)
- self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module")
+ self.assertTrue(re.search('(load|depends[-_]on).*intel/2018a', modtxt),
+ "load statement for intel/2018a found in module")
# Test we can create a module even for an installation where we don't have write permissions
os.remove(toy_core_mod)
@@ -1788,7 +1803,8 @@ def test_module_only(self):
# make sure load statements for dependencies are included
modtxt = read_file(toy_core_mod)
- self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module")
+ self.assertTrue(re.search('(load|depends[-_]on).*intel/2018a', modtxt),
+ "load statement for intel/2018a found in module")
os.remove(toy_core_mod)
os.remove(toy_mod)
@@ -1814,7 +1830,8 @@ def test_module_only(self):
# make sure load statements for dependencies are included
modtxt = read_file(toy_mod + '.lua')
- self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module")
+ self.assertTrue(re.search('(load|depends[-_]on).*intel/2018a', modtxt),
+ "load statement for intel/2018a found in module")
def test_module_only_extensions(self):
"""
@@ -2318,6 +2335,7 @@ def test_reproducibility_ext_easyblocks(self):
def test_toy_toy(self):
"""Test building two easyconfigs in a single go, with one depending on the other."""
+
topdir = os.path.dirname(os.path.abspath(__file__))
toy_ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
toy_ec_txt = read_file(toy_ec_file)
@@ -2329,9 +2347,17 @@ def test_toy_toy(self):
])
write_file(ec1, ec1_txt)
+ # adapt toy easyconfig for toy2 to produce a modulefile with a dedicated
+ # name ('toy2' instead of 'toy')
ec2 = os.path.join(self.test_prefix, 'toy2.eb')
ec2_txt = '\n'.join([
toy_ec_txt,
+ "name = 'toy2'",
+ "easyblock = 'EB_toy'",
+ "sources = ['toy/toy-0.0.tar.gz']",
+ "patches = []",
+ "sanity_check_paths = { 'files': ['bin/toy2'], 'dirs': ['bin']}",
+ "prebuildopts = 'mv toy.source toy2.c &&'",
"versionsuffix = '-two'",
"dependencies = [('toy', '0.0', '-one')]",
])
@@ -2341,7 +2367,7 @@ def test_toy_toy(self):
self._test_toy_build(ec_file=self.test_prefix, verify=False)
mod1 = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-one')
- mod2 = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-two')
+ mod2 = os.path.join(self.test_installpath, 'modules', 'all', 'toy2', '0.0-two')
if get_module_syntax() == 'Lua':
mod1 += '.lua'
mod2 += '.lua'
@@ -2350,19 +2376,19 @@ def test_toy_toy(self):
mod2_txt = read_file(mod2)
- load1_regex = re.compile('load.*toy/0.0-one', re.M)
+ load1_regex = re.compile('(load|depends[-_]on).*toy/0.0-one', re.M)
self.assertTrue(load1_regex.search(mod2_txt), "Pattern '%s' found in: %s" % (load1_regex.pattern, mod2_txt))
# Check the contents of the dumped env in the reprod dir to ensure it contains the dependency load
- reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0-two', 'easybuild', 'reprod')
- dumpenv_script = os.path.join(reprod_dir, 'toy-0.0-two.env')
+ reprod_dir = os.path.join(self.test_installpath, 'software', 'toy2', '0.0-two', 'easybuild', 'reprod')
+ dumpenv_script = os.path.join(reprod_dir, 'toy2-0.0-two.env')
reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script)
self.assertExists(reprod_dumpenv)
# Check contents of the dumpenv script
patterns = [
"""#!/bin/bash""",
- """# usage: source toy-0.0-two.env""",
+ """# usage: source toy2-0.0-two.env""",
# defining build env
"""module load toy/0.0-one""",
"""# (no build environment defined)""",
@@ -2821,8 +2847,8 @@ def grab_gcc_rpath_wrapper_args():
# test use of rpath toolchain option
test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
- toy_ec_txt = read_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb'))
- toy_ec_txt += "\ntoolchainopts = {'rpath': False}\n"
+ toy_ec_txt = read_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0-gompi-2018a.eb'))
+ toy_ec_txt += "\ntoolchainopts = {'rpath': False}\n" # overwrites existing toolchainopts
toy_ec = os.path.join(self.test_prefix, 'toy.eb')
write_file(toy_ec, toy_ec_txt)
with self.mocked_stdout_stderr():
@@ -2835,9 +2861,9 @@ def test_toy_filter_rpath_sanity_libs(self):
toy_ec = os.path.join(test_ecs, 't', 'toy-app', 'toy-app-0.0.eb')
# This should just build succesfully
- args = ['--rpath']
+ rpath_args = ['--rpath', '--strict-rpath-sanity-check']
with self.mocked_stdout_stderr():
- self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True)
+ self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=rpath_args, raise_error=True)
libtoy_libdir = os.path.join(self.test_installpath, 'software', 'libtoy', '0.0', 'lib')
toyapp_bin = os.path.join(self.test_installpath, 'software', 'toy-app', '0.0', 'bin', 'toy-app')
@@ -2858,16 +2884,16 @@ def test_toy_filter_rpath_sanity_libs(self):
# test sanity error when --rpath-filter is used to filter a required library
# In this test, libtoy.so will be linked, but not RPATH-ed due to the --rpath-filter
# Thus, the RPATH sanity check is expected to fail with libtoy.so not being found
+ args = rpath_args + ['--rpath-filter=.*libtoy.*']
error_pattern = r"Sanity check failed\: Library libtoy\.so not found"
with self.mocked_stdout_stderr():
self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=toy_ec,
- extra_args=['--rpath', '--rpath-filter=.*libtoy.*'],
- name='toy-app', raise_error=True, verbose=False)
+ extra_args=args, name='toy-app', raise_error=True, verbose=False)
# test use of --filter-rpath-sanity-libs option. In this test, we use --rpath-filter to make sure libtoy.so is
# not rpath-ed. Then, we use --filter-rpath-sanity-libs to make sure the RPATH sanity checks ignores
# the fact that libtoy.so is not found. Thus, this build should complete succesfully
- args = ['--rpath', '--rpath-filter=.*libtoy.*', '--filter-rpath-sanity-libs=libtoy.so']
+ args = rpath_args + ['--rpath-filter=.*libtoy.*', '--filter-rpath-sanity-libs=libtoy.so']
with self.mocked_stdout_stderr():
self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True)
@@ -2884,7 +2910,7 @@ def test_toy_filter_rpath_sanity_libs(self):
f"Pattern '{notfound.pattern}' should be found in: {res.output}")
# test again with list of library names passed to --filter-rpath-sanity-libs
- args = ['--rpath', '--rpath-filter=.*libtoy.*', '--filter-rpath-sanity-libs=libfoo.so,libtoy.so,libbar.so']
+ args = rpath_args + ['--rpath-filter=.*libtoy.*', '--filter-rpath-sanity-libs=libfoo.so,libtoy.so,libbar.so']
with self.mocked_stdout_stderr():
self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True)
@@ -2900,6 +2926,19 @@ def test_toy_filter_rpath_sanity_libs(self):
self.assertTrue(notfound.search(res.output),
f"Pattern '{notfound.pattern}' should be found in: {res.output}")
+ # by default, without using --strict-rpath-sanity-check, there's no failure since RPATH sanity check
+ # doesn't check for missing libraries with $LD_LIBRARY_PATH unset
+ args = ['--rpath', '--rpath-filter=.*libtoy.*']
+ with self.mocked_stdout_stderr():
+ self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True, verbose=False)
+
+ # trouble again when $LD_LIBRARY_PATH is not used in generated module file
+ args = ['libtoy-0.0.eb', '--rebuild', '--rpath', '--rpath-filter=.*libtoy.*',
+ '--filter-env-vars=LD_LIBRARY_PATH']
+ with self.mocked_stdout_stderr():
+ self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=toy_ec,
+ extra_args=args, name='toy-app', raise_error=True, verbose=False)
+
def test_toy_modaltsoftname(self):
"""Build two dependent toys as in test_toy_toy but using modaltsoftname"""
topdir = os.path.dirname(os.path.abspath(__file__))
@@ -2986,7 +3025,7 @@ def test_toy_build_trace(self):
r"\tgcc toy.c -o toy\n"
r"\t\[started at: .*\]",
r"\t\[working dir: .*\]",
- r"\t\[output saved to .*\]",
+ r"\t\[output and state saved to .*\]",
r'',
]),
r" >> command completed: exit 0, ran in .*",
@@ -3155,6 +3194,7 @@ def test_toy_multi_deps(self):
test_ec_txt += "\nexts_list = [('barbar', '1.2', {'start_dir': 'src'})]"
test_ec_txt += "\nmulti_deps = {'GCC': ['4.6.3', '7.3.0-2.30']}"
+
write_file(test_ec, test_ec_txt)
test_mod_path = os.path.join(self.test_installpath, 'modules', 'all')
@@ -3183,15 +3223,25 @@ def test_toy_multi_deps(self):
# check whether (guarded) load statement for first version listed in multi_deps is there
if get_module_syntax() == 'Lua':
expected = '\n'.join([
- 'if not ( isloaded("GCC/4.6.3") ) and not ( isloaded("GCC/7.3.0-2.30") ) then',
- ' load("GCC/4.6.3")',
+ 'if mode() == "unload" or isloaded("GCC/7.3.0-2.30") then',
+ ' depends_on("GCC")',
+ 'else',
+ ' depends_on("GCC/4.6.3")',
'end',
])
else:
+ if isinstance(self.modtool, EnvironmentModules):
+ load_stmt = "module load"
+ else:
+ load_stmt = "depends-on"
expected = '\n'.join([
- 'if { ![ is-loaded GCC/4.6.3 ] && ![ is-loaded GCC/7.3.0-2.30 ] } {',
- ' module load GCC/4.6.3',
+ '',
+ "if { [ module-info mode remove ] || [ is-loaded GCC/7.3.0-2.30 ] } {",
+ " %s GCC" % load_stmt,
+ '} else {',
+ " %s GCC/4.6.3" % load_stmt,
'}',
+ '',
])
self.assertIn(expected, toy_mod_txt)
@@ -3754,7 +3804,9 @@ def __exit__(self, type, value, traceback):
toy_ec_txt = read_file(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb'))
test_ec = os.path.join(self.test_prefix, 'test.eb')
- write_file(test_ec, toy_ec_txt + '\npostinstallcmds = ["sleep 5"]')
+ write_file(test_ec, toy_ec_txt + '\npostinstallcmds = ["sleep 10"]')
+
+ extra_args = ['--locks-dir=%s' % locks_dir, '--wait-on-lock-limit=3', '--wait-on-lock-interval=3']
signums = [
(signal.SIGABRT, SystemExit),
@@ -3767,7 +3819,7 @@ def __exit__(self, type, value, traceback):
# avoid recycling stderr of previous test
stderr = ''
- with WaitAndSignal(1, signum):
+ with WaitAndSignal(3, signum):
# change back to original working directory before each test
change_dir(orig_wd)
@@ -3775,7 +3827,7 @@ def __exit__(self, type, value, traceback):
self.mock_stderr(True)
self.mock_stdout(True)
self.assertErrorRegex(exc, '.*', self._test_toy_build, ec_file=test_ec, verify=False,
- raise_error=True, testing=False, raise_systemexit=True)
+ extra_args=extra_args, raise_error=True, testing=False, raise_systemexit=True)
stderr = self.get_stderr().strip()
self.mock_stderr(False)
@@ -3955,7 +4007,7 @@ def test_toy_build_sanity_check_linked_libs(self):
self.assertErrorRegex(EasyBuildError, error_msg, self._test_toy_build, force=False,
ec_file=test_ec, extra_args=['--module-only'], raise_error=True, verbose=False)
- # check behaviour when alternate subdirectories are specified
+ # check behaviour when alternative subdirectories are specified
test_ec_txt = read_file(libtoy_ec)
test_ec_txt += "\nbin_lib_subdirs = ['', 'lib', 'lib64']"
write_file(test_ec, test_ec_txt)
@@ -4204,6 +4256,70 @@ def test_eb_error(self):
stderr = stderr.getvalue()
self.assertTrue(regex.search(stderr), f"Pattern '{regex.pattern}' should be found in {stderr}")
+ def test_toy_python(self):
+ """
+ Test whether $PYTHONPATH or $EBPYTHONPREFIXES are set correctly.
+ """
+ # generate fake Python modules that we can use as runtime dependency for toy
+ # (required condition for use of $EBPYTHONPREFIXES)
+ fake_mods_path = os.path.join(self.test_prefix, 'modules')
+ for pyver in ('2.7', '3.6'):
+ fake_python_mod = os.path.join(fake_mods_path, 'Python', pyver)
+ if get_module_syntax() == 'Lua':
+ fake_python_mod += '.lua'
+ write_file(fake_python_mod, '')
+ else:
+ write_file(fake_python_mod, '#%Module')
+ self.modtool.use(fake_mods_path)
+
+ test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt += "\npostinstallcmds.append('mkdir -p %(installdir)s/lib/python3.6/site-packages')"
+ test_ec_txt += "\npostinstallcmds.append('touch %(installdir)s/lib/python3.6/site-packages/foo.py')"
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, test_ec_txt)
+ self.run_test_toy_build_with_output(ec_file=test_ec)
+
+ toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
+ if get_module_syntax() == 'Lua':
+ toy_mod += '.lua'
+ toy_mod_txt = read_file(toy_mod)
+
+ pythonpath_regex = re.compile('^prepend.path.*PYTHONPATH.*lib/python3.6/site-packages', re.M)
+
+ self.assertTrue(pythonpath_regex.search(toy_mod_txt),
+ f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}")
+
+ # also check when opting in to use $EBPYTHONPREFIXES instead of $PYTHONPATH
+ args = ['--prefer-python-search-path=EBPYTHONPREFIXES']
+ self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+ toy_mod_txt = read_file(toy_mod)
+ # if Python is not listed as a runtime dependency then $PYTHONPATH is still used,
+ # because the Python dependency used must be aware of $EBPYTHONPREFIXES
+ # (see sitecustomize.py installed by Python easyblock)
+ self.assertTrue(pythonpath_regex.search(toy_mod_txt),
+ f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}")
+
+ # if Python is listed as runtime dependency, then $EBPYTHONPREFIXES is used if it's preferred
+ write_file(test_ec, test_ec_txt + "\ndependencies = [('Python', '3.6', '', SYSTEM)]")
+ self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+ toy_mod_txt = read_file(toy_mod)
+
+ ebpythonprefixes_regex = re.compile('^prepend.path.*EBPYTHONPREFIXES.*root', re.M)
+ self.assertTrue(ebpythonprefixes_regex.search(toy_mod_txt),
+ f"Pattern '{ebpythonprefixes_regex.pattern}' found in: {toy_mod_txt}")
+
+ # if Python is listed in multi_deps, then $EBPYTHONPREFIXES is used, even if it's not explicitely preferred
+ write_file(test_ec, test_ec_txt + "\nmulti_deps = {'Python': ['2.7', '3.6']}")
+ self.run_test_toy_build_with_output(ec_file=test_ec)
+ toy_mod_txt = read_file(toy_mod)
+
+ self.assertTrue(ebpythonprefixes_regex.search(toy_mod_txt),
+ f"Pattern '{ebpythonprefixes_regex.pattern}' found in: {toy_mod_txt}")
+
def suite():
""" return all the tests in this file """
diff --git a/test/framework/tweak.py b/test/framework/tweak.py
index eda8046d52..24ec85b2c8 100644
--- a/test/framework/tweak.py
+++ b/test/framework/tweak.py
@@ -1,5 +1,5 @@
##
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py
index 1eb2f19fd3..ded577ef69 100644
--- a/test/framework/type_checking.py
+++ b/test/framework/type_checking.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -317,9 +317,9 @@ def test_to_toolchain_dict(self):
self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, ['gcc', '4', 'False', '7'])
# invalid truth value
- errstr = "invalid truth value .*"
- self.assertErrorRegex(ValueError, errstr, to_toolchain_dict, "intel, 2015, foo")
- self.assertErrorRegex(ValueError, errstr, to_toolchain_dict, ['gcc', '4', '7'])
+ errstr = "Invalid truth value .*"
+ self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, "intel, 2015, foo")
+ self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, ['gcc', '4', '7'])
# missing keys
self.assertErrorRegex(EasyBuildError, "Incorrect set of keys", to_toolchain_dict, {'name': 'intel'})
diff --git a/test/framework/utilities.py b/test/framework/utilities.py
index b32a5fd06a..c20de50624 100644
--- a/test/framework/utilities.py
+++ b/test/framework/utilities.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -207,8 +207,7 @@ def disallow_deprecated_behaviour(self):
def allow_deprecated_behaviour(self):
"""Restore EasyBuild version to what it was originally, to allow triggering deprecated behaviour."""
- if 'EASYBUILD_DEPRECATED' in os.environ:
- del os.environ['EASYBUILD_DEPRECATED']
+ os.environ.pop('EASYBUILD_DEPRECATED', None)
eb_build_log.CURRENT_VERSION = self.orig_current_version
@contextmanager
@@ -278,8 +277,7 @@ def reset_modulepath(self, modpaths):
# make very sure $MODULEPATH is totally empty
# some paths may be left behind, e.g. when they contain environment variables
# example: "module unuse Modules/$MODULE_VERSION/modulefiles" may not yield the desired result
- if 'MODULEPATH' in os.environ:
- del os.environ['MODULEPATH']
+ os.environ.pop('MODULEPATH', None)
for modpath in modpaths:
self.modtool.add_module_path(modpath, set_mod_paths=False)
self.modtool.set_mod_paths()
@@ -401,7 +399,7 @@ def setup_categorized_hmns_modules(self):
src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'modules', 'CategorizedHMNS', mod_subdir)
copy_dir(src_mod_path, os.path.join(mod_prefix, mod_subdir))
- # create empty module file directory to make C/Tcl modules happy
+ # create empty module file directory to make Environment Modules <5.0 happy
mpi_pref = os.path.join(mod_prefix, 'MPI', 'GCC', '6.4.0-2.28', 'OpenMPI', '2.1.2')
mkdir(os.path.join(mpi_pref, 'base'))
diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py
index ba4766f302..7a10692532 100644
--- a/test/framework/utilities_test.py
+++ b/test/framework/utilities_test.py
@@ -1,5 +1,5 @@
##
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -140,11 +140,22 @@ def test_LooseVersion(self):
self.assertLess(LooseVersion('2.1.5'), LooseVersion('2.2'))
self.assertLess(LooseVersion('2.1.3'), LooseVersion('3'))
self.assertLessEqual(LooseVersion('2.1.0'), LooseVersion('2.2'))
- # Careful here: 1.0 > 1 !!!
- self.assertGreater(LooseVersion('1.0'), LooseVersion('1'))
- self.assertLess(LooseVersion('1'), LooseVersion('1.0'))
-
- # The following test is taken from Python distutils tests
+ # Missing components are either empty strings or zeroes
+ self.assertEqual(LooseVersion('1.0'), LooseVersion('1'))
+ self.assertEqual(LooseVersion('1'), LooseVersion('1.0'))
+ self.assertEqual(LooseVersion('1.0'), LooseVersion('1.'))
+ self.assertGreater(LooseVersion('2.1.a'), LooseVersion('2.1'))
+ self.assertGreater(LooseVersion('2.a'), LooseVersion('2'))
+
+ # checking prereleases
+ version_4beta = LooseVersion('4.0.0-beta')
+ self.assertGreater(version_4beta, LooseVersion('4.0.0'))
+ self.assertTrue(version_4beta.is_prerelease('4.0.0', ['-beta']))
+ self.assertTrue(version_4beta.is_prerelease(LooseVersion('4.0.0'), ['-beta']))
+ self.assertFalse(version_4beta.is_prerelease('4.0.0', ['rc']))
+ self.assertFalse(version_4beta.is_prerelease('4.0.0', ['rc, -beta']))
+
+ # The following test is based on the Python distutils tests
# licensed under the Python Software Foundation License Version 2
versions = (('1.5.1', '1.5.2b2', -1),
('161', '3.10a', 1),
@@ -154,16 +165,21 @@ def test_LooseVersion(self):
('2g6', '11g', -1),
('0.960923', '2.2beta29', -1),
('1.13++', '5.5.kw', -1),
- # Added from https://bugs.python.org/issue14894
('a.12.b.c', 'a.b.3', -1),
- ('1.0', '1', 1),
- ('1', '1.0', -1))
+ ('1.0', '1', 0),
+ ('1.a', '1', 1),
+ )
for v1, v2, wanted in versions:
res = LooseVersion(v1)._cmp(LooseVersion(v2))
self.assertEqual(res, wanted,
'cmp(%s, %s) should be %s, got %s' %
(v1, v2, wanted, res))
+ # Test the inverse
+ res = LooseVersion(v2)._cmp(LooseVersion(v1))
+ self.assertEqual(res, -wanted,
+ 'cmp(%s, %s) should be %s, got %s' %
+ (v2, v1, -wanted, res))
# vstring is the unparsed version
self.assertEqual(LooseVersion(v1).vstring, v1)
diff --git a/test/framework/variables.py b/test/framework/variables.py
index 8331a6c2dd..cc804d6a65 100644
--- a/test/framework/variables.py
+++ b/test/framework/variables.py
@@ -1,5 +1,5 @@
# #
-# Copyright 2012-2023 Ghent University
+# Copyright 2012-2024 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),